[ Jan 16 23:21:29 Enabled. ] [ Jan 16 23:21:29 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Jan 16 23:21:29 Method "start" exited with status 0. ] [ Jan 16 23:21:29 Rereading configuration. ] [ Jan 16 23:21:29 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:29.234329321Z","hostname":"buskin","pid":1566,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:29.234487004Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:29.28741433Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:29.287450241Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:29.400533442Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:29.409635477Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:29.417799824Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.020810435Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:9fb2::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.157829416Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:126"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.157855167Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:138"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.157959129Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:151"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.15804451Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.158059741Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.158075651Z","hostname":"buskin","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:717"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.161196221Z","hostname":"buskin","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:624"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:30.161202811Z","hostname":"buskin","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:766"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.161214142Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:187"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.161226332Z","hostname":"buskin","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:825"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.161242352Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.161322144Z","hostname":"buskin","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.161403095Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.161426306Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.161434536Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.173187181Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.173217462Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"77444133-e801-4aba-9330-b96b8b00119b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.272452908Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 77444133-e801-4aba-9330-b96b8b00119b (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.628277372Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.628315302Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.628324203Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.628336463Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.628350053Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.628363003Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.628367143Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.636283635Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.636310676Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"c380568a-31f5-4b69-bfdd-8314990b56fd (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:30.721816108Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: c380568a-31f5-4b69-bfdd-8314990b56fd (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068091258Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068139309Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 77444133-e801-4aba-9330-b96b8b00119b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c380568a-31f5-4b69-bfdd-8314990b56fd (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068177479Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068186089Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.0681966Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.06820483Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.06821182Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.06822996Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068240261Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068246421Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 77444133-e801-4aba-9330-b96b8b00119b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c380568a-31f5-4b69-bfdd-8314990b56fd (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068266261Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068271551Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068275401Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068280221Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068287031Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 77444133-e801-4aba-9330-b96b8b00119b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c380568a-31f5-4b69-bfdd-8314990b56fd (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068304622Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068311052Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068315642Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068319332Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068323182Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 77444133-e801-4aba-9330-b96b8b00119b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c380568a-31f5-4b69-bfdd-8314990b56fd (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068341272Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068350453Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068355223Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068359963Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068365763Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068370043Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 77444133-e801-4aba-9330-b96b8b00119b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c380568a-31f5-4b69-bfdd-8314990b56fd (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068390973Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068396833Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068407084Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068413574Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068463605Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068472205Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068480245Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068484925Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 77444133-e801-4aba-9330-b96b8b00119b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c380568a-31f5-4b69-bfdd-8314990b56fd (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068515986Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068521406Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068525276Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068531516Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068535576Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068541176Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 77444133-e801-4aba-9330-b96b8b00119b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c380568a-31f5-4b69-bfdd-8314990b56fd (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068568677Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068573647Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068578337Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068584257Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068589367Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068609358Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 77444133-e801-4aba-9330-b96b8b00119b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c380568a-31f5-4b69-bfdd-8314990b56fd (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068644588Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068650129Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068654169Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068659309Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068663669Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 77444133-e801-4aba-9330-b96b8b00119b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c380568a-31f5-4b69-bfdd-8314990b56fd (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068693549Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068699119Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068708119Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.06871234Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:97"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.06872358Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:99"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.06872793Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 77444133-e801-4aba-9330-b96b8b00119b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c380568a-31f5-4b69-bfdd-8314990b56fd (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.06875466Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.06876Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068764221Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068768411Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:211"} {"msg":"No ledger in /pool/int/77444133-e801-4aba-9330-b96b8b00119b/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068773241Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068792251Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068801111Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/77444133-e801-4aba-9330-b96b8b00119b/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068813201Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068820232Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068825012Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068840882Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:216"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.068846242Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:231"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.079457946Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:737"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.079485417Z","hostname":"buskin","pid":1566,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999916088s","next_cleanup":"Instant { tv_sec: 1399, tv_nsec: 132316613 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.079508307Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3962"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.079528557Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.082030126Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.082077946Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.082429263Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.084070365Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:51","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: Some(\n None,\n ),\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n tx_eq: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.084268278Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1204"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.084278199Z","hostname":"buskin","pid":1566,"component":"DnsResolver","component":"RSS","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.084312729Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.084402091Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/rss-started.marker","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.084439452Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/rss-started.marker","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.084460242Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.084478862Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.084488983Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS not previously run. Creating plans.","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.084761148Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1253"} {"msg":"Writing ledger to /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/.rss-started.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.084771198Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/.rss-started.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.084974522Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.085088944Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1285"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.085093954Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:66"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.085097254Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:39"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.085127945Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1316"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.085176616Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/77444133-e801-4aba-9330-b96b8b00119b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.085191006Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.085336199Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.085494992Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.085587034Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 198593b5-c3c6-4ef5-879a-52095a2ef511 (sled), rack_id: 79114303-f678-43f7-aff3-a882c8c99560, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.085948391Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4259"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.086011472Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4659"} {"msg":"Loaded keys and certs","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.090049609Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/client.rs:50"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.095379542Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.095484634Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.099585183Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.099665674Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.101968188Z","hostname":"buskin","pid":1566,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:50386","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:74"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 198593b5-c3c6-4ef5-879a-52095a2ef511 (sled), rack_id: 79114303-f678-43f7-aff3-a882c8c99560, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.102223543Z","hostname":"buskin","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:362"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.102238234Z","hostname":"buskin","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:377"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.102302885Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/server.rs:44"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.102307825Z","hostname":"buskin","pid":1566,"component":"DnsResolver","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.102336036Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.102341876Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:813"} {"msg":"No ledger in /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.102389566Z","hostname":"buskin","pid":1566,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.102397366Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"No ledger in /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.102414187Z","hostname":"buskin","pid":1566,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.102421167Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:802"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.102424497Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:830"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.102427727Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:596"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.102469068Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.102477138Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:701","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.113301876Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.132878172Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.149555752Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.182976224Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:412"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.183023005Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.183029095Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:424"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.183034755Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 105.979304ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.186174115Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4fe7e416-4377-4af2-b520-0bceb6de640d\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:21:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4fe7e416-4377-4af2-b520-0bceb6de640d\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_77444133-e801-4aba-9330-b96b8b00119b/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.197496043Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_77444133-e801-4aba-9330-b96b8b00119b/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.221252489Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_77444133-e801-4aba-9330-b96b8b00119b/swap\"","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.22128989Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.228130431Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:439"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.228143282Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:129"} {"msg":"Ensuring dataset oxi_77444133-e801-4aba-9330-b96b8b00119b/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.228147672Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:139"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.292655071Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 111.72182ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.293211051Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"695db828-a22b-4dd0-813a-5e39b48102aa\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:21:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"695db828-a22b-4dd0-813a-5e39b48102aa\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.297964953Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:174"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.406157331Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 203.979159ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.40664487Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c96d080c-4b71-4806-b2fe-ac561b0988f4\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:21:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c96d080c-4b71-4806-b2fe-ac561b0988f4\" }"} {"msg":"Mounting oxi_77444133-e801-4aba-9330-b96b8b00119b/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.517367827Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:182"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.52952224Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:190"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.529590721Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:190"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.529602781Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:190"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.529649962Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:196"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.611733899Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 152.500278ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.612572985Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9b199217-ad6e-4717-95d7-a520c0240121\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:21:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9b199217-ad6e-4717-95d7-a520c0240121\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.766172335Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 303.548349ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:31.767092352Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ca95d0d0-b96e-427e-997c-351c7b5b6554\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:21:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ca95d0d0-b96e-427e-997c-351c7b5b6554\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:32.072127781Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 249.36451ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:32.072823064Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ebb8a1c9-c1b2-4f77-8167-485cc14aa047\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:21:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ebb8a1c9-c1b2-4f77-8167-485cc14aa047\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:32.279369691Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:93"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:32.285179002Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:32.285218223Z","hostname":"buskin","pid":1566,"component":"internal-dns-resolver","file":"internal-dns/resolver/src/resolver.rs:111","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:32.285279594Z","hostname":"buskin","pid":1566,"file":"oximeter/producer/src/lib.rs:279","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/producer/src/lib.rs:279","interval":"30s","address":"[fd00:1122:3344:101::1]:42407","producer_id":"198593b5-c3c6-4ef5-879a-52095a2ef511"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:32.286456777Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"359.721922ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:32.287275462Z","hostname":"buskin","pid":1566,"component":"VmmReservoirManager","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:32.287282393Z","hostname":"buskin","pid":1566,"component":"VmmReservoirManager","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:32.287322934Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:921","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:32.287328523Z","hostname":"buskin","pid":1566,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:32.323419157Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:32.647911709Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"340.493813ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:32.990713251Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"727.968404ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:33.720407436Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.220324507s"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:34.756676807Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2683"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:34.756709308Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2928"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:34.756713848Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2553"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:34.756717538Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2600"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:34.756729248Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:34.756732988Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:34.758590724Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2853"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:34.759792537Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2594"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:34.75990628Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:34.760040162Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 33.158377ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:35.027408646Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:35.028465567Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.450953614s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 117.853086ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:35.229751503Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 169.910665ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:35.428668973Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 207.436191ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:35.627488642Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 831.584733ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:35.979732036Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.476828189s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:36.940869675Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:37.481924256Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"10.643291947s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.319178745s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:38.449930696Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.370126204s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:39.809615769Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/198593b5-c3c6-4ef5-879a-52095a2ef511)","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:41.495602758Z","hostname":"buskin","pid":1566,"component":"NexusNotifierTask","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:41.525933531Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:41.526441241Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12348","component":"dropshot (Repo Depot)","component":"ArtifactStore","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:41.527829527Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:41.527854378Z","hostname":"buskin","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:432"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:41.527865848Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Writing ledger to /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:41.527889119Z","hostname":"buskin","pid":1566,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:41.528113923Z","hostname":"buskin","pid":1566,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:41.528321687Z","hostname":"buskin","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:41.528367528Z","hostname":"buskin","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.1.100, vlan_id: None, rib_priority: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: Some(None), bgp_peers: [], autoneg: false, lldp: None, tx_eq: None }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:41.528391448Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3962"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:41.528400258Z","hostname":"buskin","pid":1566,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:50386","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:81"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:41.713630836Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:117","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:41.713706857Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:252","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:41.718734574Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4283","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:41.718752004Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4299"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:41.718758784Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:534"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:41.987121328Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:63904"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:41.987389833Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:41.987398083Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:41.987463284Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:41.987477605Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:41.987620017Z","hostname":"buskin","pid":1566,"uri":"/inventory","method":"GET","req_id":"2ee2f269-926c-4903-b287-47df4878666d","remote_addr":"[fd00:1122:3344:101::1]:63904","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":343,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.019372837Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:63664"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.019542641Z","hostname":"buskin","pid":1566,"uri":"/sled-role","method":"GET","req_id":"d9556618-103f-4390-bf49-0cd9b4a3751a","remote_addr":"[fd00:1122:3344:101::1]:63664","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":28,"response_code":"200"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(2), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 39c64568-3037-4b5e-b5cd-f3b5bbb00c23 (physical_disk), pool_id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 3ac42834-12e9-42f4-a50c-f6c5e6e15b57 (physical_disk), pool_id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 4dc50e8c-89b7-4650-b83d-2afc1f95c576 (physical_disk), pool_id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 4f9bd4b9-9e0b-49e5-a28c-b4edc0ed7598 (physical_disk), pool_id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 6c1201e9-1f94-4208-a20b-1a1c197a0cf3 (physical_disk), pool_id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 6ff9e48f-003f-46bc-a950-4db730adca16 (physical_disk), pool_id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 89bc463a-6460-4fc8-baa9-baa3a411f65f (physical_disk), pool_id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: a1960b6f-2e08-488f-800f-0f869598945a (physical_disk), pool_id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: e0f267d4-aab3-4256-8aba-cabafa6e1c7e (physical_disk), pool_id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.050206539Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:385"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.050310681Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:50323"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.050659878Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:893"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(2), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 39c64568-3037-4b5e-b5cd-f3b5bbb00c23 (physical_disk), pool_id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 3ac42834-12e9-42f4-a50c-f6c5e6e15b57 (physical_disk), pool_id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 4dc50e8c-89b7-4650-b83d-2afc1f95c576 (physical_disk), pool_id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 4f9bd4b9-9e0b-49e5-a28c-b4edc0ed7598 (physical_disk), pool_id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 6c1201e9-1f94-4208-a20b-1a1c197a0cf3 (physical_disk), pool_id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 6ff9e48f-003f-46bc-a950-4db730adca16 (physical_disk), pool_id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 89bc463a-6460-4fc8-baa9-baa3a411f65f (physical_disk), pool_id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: a1960b6f-2e08-488f-800f-0f869598945a (physical_disk), pool_id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: e0f267d4-aab3-4256-8aba-cabafa6e1c7e (physical_disk), pool_id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.050717569Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.050726949Z","hostname":"buskin","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.05073592Z","hostname":"buskin","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.05074253Z","hostname":"buskin","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1298"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.05076501Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.05076939Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.05077373Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.050784461Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.183244314Z","hostname":"buskin","pid":1566,"component":"VmmReservoirManager","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.190500214Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.190511754Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.284800155Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-16T23:21:42.289947254Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15343543213510289921"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-16T23:21:42.290050266Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"7841044830787084053"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.293176086Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.293244617Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.293251477Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.293257157Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.293263978Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.29338982Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.29339719Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.328088916Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.336038089Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.476063148Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.485354337Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.485386857Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.485438768Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.485447258Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.485453039Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.495892809Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.49591792Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.593669587Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.601607049Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.60165861Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.60166599Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.601827674Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.601836704Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.601922095Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.601928566Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/198593b5-c3c6-4ef5-879a-52095a2ef511)","v":0,"name":"SledAgent","level":50,"time":"2025-01-16T23:21:42.606121406Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.635652773Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.642621407Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_5db46ce3-4889-4034-8561-7e871bb98ae0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.779862823Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.788334846Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.788369686Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.788384347Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.788389607Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.788395547Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.798233216Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.798259256Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.89060413Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.898235406Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.898272867Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.898281007Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.898328588Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.898339299Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.898520732Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.898529552Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.934019584Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_736d5541-d5bd-460c-a5f5-441d6996aa03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:42.941961666Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_736d5541-d5bd-460c-a5f5-441d6996aa03","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.080622788Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.088839676Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.088879027Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.088892598Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.088899138Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.088905028Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.098259927Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.098289068Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.193931565Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.202128262Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.202162603Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.202178273Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.202239874Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.202250635Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.202387497Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.202396337Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.236053434Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_65ae8c88-c7ba-40ff-af14-085d448fc3c0/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.243791892Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_65ae8c88-c7ba-40ff-af14-085d448fc3c0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.381808673Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.389636023Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.389667734Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.389672024Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.389677434Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.389688455Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.400201376Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.400241977Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"9764f33b-5a77-475e-83b5-777615774b6a (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.49253723Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.500192677Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.500256728Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.500313639Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.5003703Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.50038119Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.500531133Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.500541673Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.53576799Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_9764f33b-5a77-475e-83b5-777615774b6a/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.544363865Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/198593b5-c3c6-4ef5-879a-52095a2ef511)","v":0,"name":"SledAgent","level":50,"time":"2025-01-16T23:21:43.608369654Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Looking for unencrypted datasets in oxp_9764f33b-5a77-475e-83b5-777615774b6a","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.680379437Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.688505934Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.688540224Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.688551084Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.688558015Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.688562204Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.698493955Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.698518516Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.789420232Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.797173941Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.797206511Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.797214191Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.797252952Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.797297333Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.797483927Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.797493127Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.832842475Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_28fb7d98-1494-4570-b25a-aa56857fbe19/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.84195657Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_28fb7d98-1494-4570-b25a-aa56857fbe19","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.983418837Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.991557303Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.991591674Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.991598964Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.991605624Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:43.991640864Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.000974954Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.000999094Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"723d2041-12e1-48e9-aa34-7148357fe80c (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.094935968Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.102979943Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.103027574Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.103033944Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.103084625Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.103095985Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.103274548Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.103283119Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.141578764Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.150856722Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_723d2041-12e1-48e9-aa34-7148357fe80c","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.294790267Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.302708549Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.302741009Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.30276346Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.30277095Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.30277689Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.313027967Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.313058648Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.408515991Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.417288139Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.41732408Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.417363191Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.417406232Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.417417412Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.417561615Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.417570355Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.453561296Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_7980b762-6ad3-435b-bf8d-91d18dcf75d4/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.461850115Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_7980b762-6ad3-435b-bf8d-91d18dcf75d4","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.608538202Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/198593b5-c3c6-4ef5-879a-52095a2ef511)","v":0,"name":"SledAgent","level":50,"time":"2025-01-16T23:21:44.612285624Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.616775641Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.616786671Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.616810791Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.616817671Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.616821711Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.626224722Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.626250372Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.72295842Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.730931873Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.730976184Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.730983444Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.730997914Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.731011484Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.731155137Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.731166378Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.767000186Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.776543029Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.916390895Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.92448068Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.924526521Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.924584812Z","hostname":"buskin","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.924732725Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(2), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 77444133-e801-4aba-9330-b96b8b00119b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c380568a-31f5-4b69-bfdd-8314990b56fd (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.924791246Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.924797586Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.924875608Z","hostname":"buskin","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.925087242Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:897"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.925101382Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.925116832Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:923","generation":"Generation(2)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.925124433Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:928"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.925133883Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:933"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.925167524Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:122","gen":"Generation(2)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.925174704Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:939"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.925211024Z","hostname":"buskin","pid":1566,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:799","gen":"Generation(2)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.925224405Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:944"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.925314496Z","hostname":"buskin","pid":1566,"uri":"/omicron-physical-disks","method":"PUT","req_id":"0d93a0cb-674e-4057-ba79-18790a645b7b","remote_addr":"[fd00:1122:3344:101::1]:50323","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":2874887,"response_code":"200"} {"msg":"attempting to set up sled's datasets: DatasetsConfig { generation: Generation(1), datasets: {08d12670-ceda-4948-a06d-7bd47f4cc54d (dataset): DatasetConfig { id: 08d12670-ceda-4948-a06d-7bd47f4cc54d (dataset), name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 16f8dfc9-25e2-419e-9b27-8c6b2b8c6c79 (dataset): DatasetConfig { id: 16f8dfc9-25e2-419e-9b27-8c6b2b8c6c79 (dataset), name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 184837d2-9214-4b98-987f-f9ce5785045a (dataset): DatasetConfig { id: 184837d2-9214-4b98-987f-f9ce5785045a (dataset), name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1b5db06b-d168-46db-b959-e38790a599b4 (dataset): DatasetConfig { id: 1b5db06b-d168-46db-b959-e38790a599b4 (dataset), name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 1c0b88c9-53f4-457a-aa80-e89fbaaca94f (dataset): DatasetConfig { id: 1c0b88c9-53f4-457a-aa80-e89fbaaca94f (dataset), name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1ed891d6-60d2-4acb-b3be-3bea6dc98c0a (dataset): DatasetConfig { id: 1ed891d6-60d2-4acb-b3be-3bea6dc98c0a (dataset), name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 1fa4060a-93b2-4e3a-be52-aa3041d850dd (dataset): DatasetConfig { id: 1fa4060a-93b2-4e3a-be52-aa3041d850dd (dataset), name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 205ca330-9980-48c3-80ac-1b2d63d5bf5f (dataset): DatasetConfig { id: 205ca330-9980-48c3-80ac-1b2d63d5bf5f (dataset), name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 2316843e-0eff-42b0-856b-eaa852c43e7d (dataset): DatasetConfig { id: 2316843e-0eff-42b0-856b-eaa852c43e7d (dataset), name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 248bc01d-4db8-4b5a-95f6-e52df354e625 (dataset): DatasetConfig { id: 248bc01d-4db8-4b5a-95f6-e52df354e625 (dataset), name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2664c935-3187-4bc6-8e2f-80fbb4967e0d (dataset): DatasetConfig { id: 2664c935-3187-4bc6-8e2f-80fbb4967e0d (dataset), name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2806b7c7-29f3-45fc-ad06-a71de63016a1 (dataset): DatasetConfig { id: 2806b7c7-29f3-45fc-ad06-a71de63016a1 (dataset), name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2c395376-b777-4e15-b6d2-794cd1c54b0d (dataset): DatasetConfig { id: 2c395376-b777-4e15-b6d2-794cd1c54b0d (dataset), name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2e346a03-db55-41a5-a70b-d23c35a97f34 (dataset): DatasetConfig { id: 2e346a03-db55-41a5-a70b-d23c35a97f34 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3840c8d8-9d20-4737-829a-7854eac84b56 (dataset): DatasetConfig { id: 3840c8d8-9d20-4737-829a-7854eac84b56 (dataset), name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3f212d18-1c99-4107-aedf-14b4c2a282aa (dataset): DatasetConfig { id: 3f212d18-1c99-4107-aedf-14b4c2a282aa (dataset), name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 41766bb4-bcf0-40a0-b85d-00102f8ef53a (dataset): DatasetConfig { id: 41766bb4-bcf0-40a0-b85d-00102f8ef53a (dataset), name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 444fa3b9-de8c-44eb-bdf7-16aef052fb35 (dataset): DatasetConfig { id: 444fa3b9-de8c-44eb-bdf7-16aef052fb35 (dataset), name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4a11653f-ec37-4d97-9093-46b112737011 (dataset): DatasetConfig { id: 4a11653f-ec37-4d97-9093-46b112737011 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4a96379b-0520-413d-883c-d7379e875519 (dataset): DatasetConfig { id: 4a96379b-0520-413d-883c-d7379e875519 (dataset), name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4bcacffb-9d91-4d0f-8b7a-86a80bed5453 (dataset): DatasetConfig { id: 4bcacffb-9d91-4d0f-8b7a-86a80bed5453 (dataset), name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 50c1d5c5-30b2-4b5d-ae8a-c0e3350b6dc3 (dataset): DatasetConfig { id: 50c1d5c5-30b2-4b5d-ae8a-c0e3350b6dc3 (dataset), name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 513c238f-74b9-40ee-a14e-9727c3e63a14 (dataset): DatasetConfig { id: 513c238f-74b9-40ee-a14e-9727c3e63a14 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5c52fae5-fc8a-4a23-adfc-c632dd4b6a8e (dataset): DatasetConfig { id: 5c52fae5-fc8a-4a23-adfc-c632dd4b6a8e (dataset), name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5df55524-b2b2-49ff-b667-ffd4c7c0d9e1 (dataset): DatasetConfig { id: 5df55524-b2b2-49ff-b667-ffd4c7c0d9e1 (dataset), name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6161fa91-227e-4e15-ac71-ea1861890ec0 (dataset): DatasetConfig { id: 6161fa91-227e-4e15-ac71-ea1861890ec0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7205aeb2-722c-456a-84c8-4f743afc7570 (dataset): DatasetConfig { id: 7205aeb2-722c-456a-84c8-4f743afc7570 (dataset), name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 727ee0d0-c51a-4273-90c1-6d2f70897edc (dataset): DatasetConfig { id: 727ee0d0-c51a-4273-90c1-6d2f70897edc (dataset), name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 74dcc54e-32ae-4e41-88da-bc00e2529565 (dataset): DatasetConfig { id: 74dcc54e-32ae-4e41-88da-bc00e2529565 (dataset), name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 769ca436-bd56-4e1d-89b2-24b9e2d1808f (dataset): DatasetConfig { id: 769ca436-bd56-4e1d-89b2-24b9e2d1808f (dataset), name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 772531c0-033b-43df-96de-bb4400ec664f (dataset): DatasetConfig { id: 772531c0-033b-43df-96de-bb4400ec664f (dataset), name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 77aee79d-6fc1-4437-a613-08e3f32d7d9e (dataset): DatasetConfig { id: 77aee79d-6fc1-4437-a613-08e3f32d7d9e (dataset), name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7c85ebae-dad3-4cbc-8eb1-d7e78ac20b48 (dataset): DatasetConfig { id: 7c85ebae-dad3-4cbc-8eb1-d7e78ac20b48 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7e736aa6-5668-4bcf-be81-b6eb9a686305 (dataset): DatasetConfig { id: 7e736aa6-5668-4bcf-be81-b6eb9a686305 (dataset), name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7f636496-9c69-4f68-87d9-0784595a1770 (dataset): DatasetConfig { id: 7f636496-9c69-4f68-87d9-0784595a1770 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7f657da3-0ee6-4b14-8330-2418806b6147 (dataset): DatasetConfig { id: 7f657da3-0ee6-4b14-8330-2418806b6147 (dataset), name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 87545144-3490-477d-8dc1-717c1ade207c (dataset): DatasetConfig { id: 87545144-3490-477d-8dc1-717c1ade207c (dataset), name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8b4484cb-bee1-4e74-b8b8-34052724ab3f (dataset): DatasetConfig { id: 8b4484cb-bee1-4e74-b8b8-34052724ab3f (dataset), name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8e8dad3b-1c66-4d33-932b-551be2ded9b8 (dataset): DatasetConfig { id: 8e8dad3b-1c66-4d33-932b-551be2ded9b8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 9235de26-fa9d-4acb-b378-702ea4804a9c (dataset): DatasetConfig { id: 9235de26-fa9d-4acb-b378-702ea4804a9c (dataset), name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 960ffcca-def8-464c-9564-225edca5dfe8 (dataset): DatasetConfig { id: 960ffcca-def8-464c-9564-225edca5dfe8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a421c94c-c09d-4494-85e8-ed37ec584e9c (dataset): DatasetConfig { id: a421c94c-c09d-4494-85e8-ed37ec584e9c (dataset), name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ac829c28-cd08-4fbb-b874-76818dd2ad3a (dataset): DatasetConfig { id: ac829c28-cd08-4fbb-b874-76818dd2ad3a (dataset), name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: TransientZone { name: \"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ae1057ab-e517-451f-9f62-87f7ab8f1d22 (dataset): DatasetConfig { id: ae1057ab-e517-451f-9f62-87f7ab8f1d22 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, aeb8435a-6da9-4ea4-9878-d4b2d17ead87 (dataset): DatasetConfig { id: aeb8435a-6da9-4ea4-9878-d4b2d17ead87 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, aefdb534-1a28-4677-ba81-137d32a09164 (dataset): DatasetConfig { id: aefdb534-1a28-4677-ba81-137d32a09164 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b1ec5514-e4c2-4994-a49b-cc4599c57d99 (dataset): DatasetConfig { id: b1ec5514-e4c2-4994-a49b-cc4599c57d99 (dataset), name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b73030f6-08fc-43fa-bb33-bc4e29b20309 (dataset): DatasetConfig { id: b73030f6-08fc-43fa-bb33-bc4e29b20309 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c371bfcb-5a89-421c-bffc-86c657d1001d (dataset): DatasetConfig { id: c371bfcb-5a89-421c-bffc-86c657d1001d (dataset), name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c6a7c94e-bbad-4790-9abe-2bf65fa318b9 (dataset): DatasetConfig { id: c6a7c94e-bbad-4790-9abe-2bf65fa318b9 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ce8ffc4e-9dc9-4c84-9bae-961692a63993 (dataset): DatasetConfig { id: ce8ffc4e-9dc9-4c84-9bae-961692a63993 (dataset), name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, cec9f810-d8c9-4939-847c-19a2b60486b3 (dataset): DatasetConfig { id: cec9f810-d8c9-4939-847c-19a2b60486b3 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d11622eb-5638-47f7-8825-3a4be2483c0c (dataset): DatasetConfig { id: d11622eb-5638-47f7-8825-3a4be2483c0c (dataset), name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d48924e7-8824-495b-8b90-7e6546f10fc2 (dataset): DatasetConfig { id: d48924e7-8824-495b-8b90-7e6546f10fc2 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d84ac5b3-7641-4d03-bac4-a812bc5a5fd8 (dataset): DatasetConfig { id: d84ac5b3-7641-4d03-bac4-a812bc5a5fd8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d87dfdb4-1b68-48de-9af4-0fda85fa260f (dataset): DatasetConfig { id: d87dfdb4-1b68-48de-9af4-0fda85fa260f (dataset), name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: TransientZone { name: \"oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, dccfa740-76ef-469c-948b-864799bdde99 (dataset): DatasetConfig { id: dccfa740-76ef-469c-948b-864799bdde99 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, de0adcca-c60a-499c-87c6-39b2086ddd8c (dataset): DatasetConfig { id: de0adcca-c60a-499c-87c6-39b2086ddd8c (dataset), name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e537bd0c-0e5e-42b2-8f97-44406b6ef38f (dataset): DatasetConfig { id: e537bd0c-0e5e-42b2-8f97-44406b6ef38f (dataset), name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e7c84b5b-636b-4488-b47f-5d982b57daf0 (dataset): DatasetConfig { id: e7c84b5b-636b-4488-b47f-5d982b57daf0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f1337a85-5e5e-4f54-aef8-527af41ca66f (dataset): DatasetConfig { id: f1337a85-5e5e-4f54-aef8-527af41ca66f (dataset), name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f291f393-fbd3-4119-a4ea-cadab4dea690 (dataset): DatasetConfig { id: f291f393-fbd3-4119-a4ea-cadab4dea690 (dataset), name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f4fd91e5-a460-442d-9f48-3ce718a71b0a (dataset): DatasetConfig { id: f4fd91e5-a460-442d-9f48-3ce718a71b0a (dataset), name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f5314705-d3b4-4106-b2bc-bcee8118cd21 (dataset): DatasetConfig { id: f5314705-d3b4-4106-b2bc-bcee8118cd21 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f9e6172c-9e7e-41e5-8808-16e9ce9685fd (dataset): DatasetConfig { id: f9e6172c-9e7e-41e5-8808-16e9ce9685fd (dataset), name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fa90cf72-ac1d-4731-af41-f4764508fe23 (dataset): DatasetConfig { id: fa90cf72-ac1d-4731-af41-f4764508fe23 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }} }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.957824681Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:460"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.957951193Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:50430"} {"msg":"datasets ensure","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.958575605Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:859"} {"msg":"Received DatasetsEnsure { config: DatasetsConfig { generation: Generation(1), datasets: {08d12670-ceda-4948-a06d-7bd47f4cc54d (dataset): DatasetConfig { id: 08d12670-ceda-4948-a06d-7bd47f4cc54d (dataset), name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 16f8dfc9-25e2-419e-9b27-8c6b2b8c6c79 (dataset): DatasetConfig { id: 16f8dfc9-25e2-419e-9b27-8c6b2b8c6c79 (dataset), name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 184837d2-9214-4b98-987f-f9ce5785045a (dataset): DatasetConfig { id: 184837d2-9214-4b98-987f-f9ce5785045a (dataset), name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1b5db06b-d168-46db-b959-e38790a599b4 (dataset): DatasetConfig { id: 1b5db06b-d168-46db-b959-e38790a599b4 (dataset), name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 1c0b88c9-53f4-457a-aa80-e89fbaaca94f (dataset): DatasetConfig { id: 1c0b88c9-53f4-457a-aa80-e89fbaaca94f (dataset), name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1ed891d6-60d2-4acb-b3be-3bea6dc98c0a (dataset): DatasetConfig { id: 1ed891d6-60d2-4acb-b3be-3bea6dc98c0a (dataset), name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 1fa4060a-93b2-4e3a-be52-aa3041d850dd (dataset): DatasetConfig { id: 1fa4060a-93b2-4e3a-be52-aa3041d850dd (dataset), name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 205ca330-9980-48c3-80ac-1b2d63d5bf5f (dataset): DatasetConfig { id: 205ca330-9980-48c3-80ac-1b2d63d5bf5f (dataset), name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 2316843e-0eff-42b0-856b-eaa852c43e7d (dataset): DatasetConfig { id: 2316843e-0eff-42b0-856b-eaa852c43e7d (dataset), name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 248bc01d-4db8-4b5a-95f6-e52df354e625 (dataset): DatasetConfig { id: 248bc01d-4db8-4b5a-95f6-e52df354e625 (dataset), name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2664c935-3187-4bc6-8e2f-80fbb4967e0d (dataset): DatasetConfig { id: 2664c935-3187-4bc6-8e2f-80fbb4967e0d (dataset), name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2806b7c7-29f3-45fc-ad06-a71de63016a1 (dataset): DatasetConfig { id: 2806b7c7-29f3-45fc-ad06-a71de63016a1 (dataset), name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2c395376-b777-4e15-b6d2-794cd1c54b0d (dataset): DatasetConfig { id: 2c395376-b777-4e15-b6d2-794cd1c54b0d (dataset), name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2e346a03-db55-41a5-a70b-d23c35a97f34 (dataset): DatasetConfig { id: 2e346a03-db55-41a5-a70b-d23c35a97f34 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3840c8d8-9d20-4737-829a-7854eac84b56 (dataset): DatasetConfig { id: 3840c8d8-9d20-4737-829a-7854eac84b56 (dataset), name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3f212d18-1c99-4107-aedf-14b4c2a282aa (dataset): DatasetConfig { id: 3f212d18-1c99-4107-aedf-14b4c2a282aa (dataset), name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 41766bb4-bcf0-40a0-b85d-00102f8ef53a (dataset): DatasetConfig { id: 41766bb4-bcf0-40a0-b85d-00102f8ef53a (dataset), name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 444fa3b9-de8c-44eb-bdf7-16aef052fb35 (dataset): DatasetConfig { id: 444fa3b9-de8c-44eb-bdf7-16aef052fb35 (dataset), name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4a11653f-ec37-4d97-9093-46b112737011 (dataset): DatasetConfig { id: 4a11653f-ec37-4d97-9093-46b112737011 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4a96379b-0520-413d-883c-d7379e875519 (dataset): DatasetConfig { id: 4a96379b-0520-413d-883c-d7379e875519 (dataset), name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4bcacffb-9d91-4d0f-8b7a-86a80bed5453 (dataset): DatasetConfig { id: 4bcacffb-9d91-4d0f-8b7a-86a80bed5453 (dataset), name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 50c1d5c5-30b2-4b5d-ae8a-c0e3350b6dc3 (dataset): DatasetConfig { id: 50c1d5c5-30b2-4b5d-ae8a-c0e3350b6dc3 (dataset), name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 513c238f-74b9-40ee-a14e-9727c3e63a14 (dataset): DatasetConfig { id: 513c238f-74b9-40ee-a14e-9727c3e63a14 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5c52fae5-fc8a-4a23-adfc-c632dd4b6a8e (dataset): DatasetConfig { id: 5c52fae5-fc8a-4a23-adfc-c632dd4b6a8e (dataset), name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5df55524-b2b2-49ff-b667-ffd4c7c0d9e1 (dataset): DatasetConfig { id: 5df55524-b2b2-49ff-b667-ffd4c7c0d9e1 (dataset), name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6161fa91-227e-4e15-ac71-ea1861890ec0 (dataset): DatasetConfig { id: 6161fa91-227e-4e15-ac71-ea1861890ec0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7205aeb2-722c-456a-84c8-4f743afc7570 (dataset): DatasetConfig { id: 7205aeb2-722c-456a-84c8-4f743afc7570 (dataset), name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 727ee0d0-c51a-4273-90c1-6d2f70897edc (dataset): DatasetConfig { id: 727ee0d0-c51a-4273-90c1-6d2f70897edc (dataset), name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 74dcc54e-32ae-4e41-88da-bc00e2529565 (dataset): DatasetConfig { id: 74dcc54e-32ae-4e41-88da-bc00e2529565 (dataset), name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 769ca436-bd56-4e1d-89b2-24b9e2d1808f (dataset): DatasetConfig { id: 769ca436-bd56-4e1d-89b2-24b9e2d1808f (dataset), name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 772531c0-033b-43df-96de-bb4400ec664f (dataset): DatasetConfig { id: 772531c0-033b-43df-96de-bb4400ec664f (dataset), name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 77aee79d-6fc1-4437-a613-08e3f32d7d9e (dataset): DatasetConfig { id: 77aee79d-6fc1-4437-a613-08e3f32d7d9e (dataset), name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7c85ebae-dad3-4cbc-8eb1-d7e78ac20b48 (dataset): DatasetConfig { id: 7c85ebae-dad3-4cbc-8eb1-d7e78ac20b48 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7e736aa6-5668-4bcf-be81-b6eb9a686305 (dataset): DatasetConfig { id: 7e736aa6-5668-4bcf-be81-b6eb9a686305 (dataset), name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7f636496-9c69-4f68-87d9-0784595a1770 (dataset): DatasetConfig { id: 7f636496-9c69-4f68-87d9-0784595a1770 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7f657da3-0ee6-4b14-8330-2418806b6147 (dataset): DatasetConfig { id: 7f657da3-0ee6-4b14-8330-2418806b6147 (dataset), name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 87545144-3490-477d-8dc1-717c1ade207c (dataset): DatasetConfig { id: 87545144-3490-477d-8dc1-717c1ade207c (dataset), name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8b4484cb-bee1-4e74-b8b8-34052724ab3f (dataset): DatasetConfig { id: 8b4484cb-bee1-4e74-b8b8-34052724ab3f (dataset), name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8e8dad3b-1c66-4d33-932b-551be2ded9b8 (dataset): DatasetConfig { id: 8e8dad3b-1c66-4d33-932b-551be2ded9b8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 9235de26-fa9d-4acb-b378-702ea4804a9c (dataset): DatasetConfig { id: 9235de26-fa9d-4acb-b378-702ea4804a9c (dataset), name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 960ffcca-def8-464c-9564-225edca5dfe8 (dataset): DatasetConfig { id: 960ffcca-def8-464c-9564-225edca5dfe8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a421c94c-c09d-4494-85e8-ed37ec584e9c (dataset): DatasetConfig { id: a421c94c-c09d-4494-85e8-ed37ec584e9c (dataset), name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ac829c28-cd08-4fbb-b874-76818dd2ad3a (dataset): DatasetConfig { id: ac829c28-cd08-4fbb-b874-76818dd2ad3a (dataset), name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: TransientZone { name: \"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ae1057ab-e517-451f-9f62-87f7ab8f1d22 (dataset): DatasetConfig { id: ae1057ab-e517-451f-9f62-87f7ab8f1d22 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, aeb8435a-6da9-4ea4-9878-d4b2d17ead87 (dataset): DatasetConfig { id: aeb8435a-6da9-4ea4-9878-d4b2d17ead87 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, aefdb534-1a28-4677-ba81-137d32a09164 (dataset): DatasetConfig { id: aefdb534-1a28-4677-ba81-137d32a09164 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b1ec5514-e4c2-4994-a49b-cc4599c57d99 (dataset): DatasetConfig { id: b1ec5514-e4c2-4994-a49b-cc4599c57d99 (dataset), name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b73030f6-08fc-43fa-bb33-bc4e29b20309 (dataset): DatasetConfig { id: b73030f6-08fc-43fa-bb33-bc4e29b20309 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c371bfcb-5a89-421c-bffc-86c657d1001d (dataset): DatasetConfig { id: c371bfcb-5a89-421c-bffc-86c657d1001d (dataset), name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c6a7c94e-bbad-4790-9abe-2bf65fa318b9 (dataset): DatasetConfig { id: c6a7c94e-bbad-4790-9abe-2bf65fa318b9 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ce8ffc4e-9dc9-4c84-9bae-961692a63993 (dataset): DatasetConfig { id: ce8ffc4e-9dc9-4c84-9bae-961692a63993 (dataset), name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, cec9f810-d8c9-4939-847c-19a2b60486b3 (dataset): DatasetConfig { id: cec9f810-d8c9-4939-847c-19a2b60486b3 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d11622eb-5638-47f7-8825-3a4be2483c0c (dataset): DatasetConfig { id: d11622eb-5638-47f7-8825-3a4be2483c0c (dataset), name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d48924e7-8824-495b-8b90-7e6546f10fc2 (dataset): DatasetConfig { id: d48924e7-8824-495b-8b90-7e6546f10fc2 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d84ac5b3-7641-4d03-bac4-a812bc5a5fd8 (dataset): DatasetConfig { id: d84ac5b3-7641-4d03-bac4-a812bc5a5fd8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d87dfdb4-1b68-48de-9af4-0fda85fa260f (dataset): DatasetConfig { id: d87dfdb4-1b68-48de-9af4-0fda85fa260f (dataset), name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: TransientZone { name: \"oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, dccfa740-76ef-469c-948b-864799bdde99 (dataset): DatasetConfig { id: dccfa740-76ef-469c-948b-864799bdde99 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, de0adcca-c60a-499c-87c6-39b2086ddd8c (dataset): DatasetConfig { id: de0adcca-c60a-499c-87c6-39b2086ddd8c (dataset), name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e537bd0c-0e5e-42b2-8f97-44406b6ef38f (dataset): DatasetConfig { id: e537bd0c-0e5e-42b2-8f97-44406b6ef38f (dataset), name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e7c84b5b-636b-4488-b47f-5d982b57daf0 (dataset): DatasetConfig { id: e7c84b5b-636b-4488-b47f-5d982b57daf0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f1337a85-5e5e-4f54-aef8-527af41ca66f (dataset): DatasetConfig { id: f1337a85-5e5e-4f54-aef8-527af41ca66f (dataset), name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f291f393-fbd3-4119-a4ea-cadab4dea690 (dataset): DatasetConfig { id: f291f393-fbd3-4119-a4ea-cadab4dea690 (dataset), name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f4fd91e5-a460-442d-9f48-3ce718a71b0a (dataset): DatasetConfig { id: f4fd91e5-a460-442d-9f48-3ce718a71b0a (dataset), name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f5314705-d3b4-4106-b2bc-bcee8118cd21 (dataset): DatasetConfig { id: f5314705-d3b4-4106-b2bc-bcee8118cd21 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f9e6172c-9e7e-41e5-8808-16e9ce9685fd (dataset): DatasetConfig { id: f9e6172c-9e7e-41e5-8808-16e9ce9685fd (dataset), name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fa90cf72-ac1d-4731-af41-f4764508fe23 (dataset): DatasetConfig { id: fa90cf72-ac1d-4731-af41-f4764508fe23 (dataset), name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }} }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.958891131Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.958897881Z","hostname":"buskin","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.958903212Z","hostname":"buskin","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested datasets', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.958907652Z","hostname":"buskin","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:909"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.974035011Z","hostname":"buskin","pid":1566,"name":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.974074752Z","hostname":"buskin","pid":1566,"name":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:44.974079392Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.022452201Z","hostname":"buskin","pid":1566,"name":"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.022486142Z","hostname":"buskin","pid":1566,"name":"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.022490412Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.072297858Z","hostname":"buskin","pid":1566,"name":"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.072334389Z","hostname":"buskin","pid":1566,"name":"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.072339479Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.121383481Z","hostname":"buskin","pid":1566,"name":"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.121423572Z","hostname":"buskin","pid":1566,"name":"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.121428462Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/723d2041-12e1-48e9-aa34-7148357fe80c/crypt/debug\"), full_name: \"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.149164815Z","hostname":"buskin","pid":1566,"name":"oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9/crypt/zone/oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.149201835Z","hostname":"buskin","pid":1566,"name":"oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9/crypt/zone/oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.149205886Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/47f34c1f-e641-403f-804d-b2c26f2cb1f9/crypt/zone/oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\"), full_name: \"oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9/crypt/zone/oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.232923394Z","hostname":"buskin","pid":1566,"name":"oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.232964964Z","hostname":"buskin","pid":1566,"name":"oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.232969804Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/47f34c1f-e641-403f-804d-b2c26f2cb1f9/crypt/debug\"), full_name: \"oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.264090792Z","hostname":"buskin","pid":1566,"name":"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.264126893Z","hostname":"buskin","pid":1566,"name":"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.264131853Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/9764f33b-5a77-475e-83b5-777615774b6a/crypt/debug\"), full_name: \"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.294083828Z","hostname":"buskin","pid":1566,"name":"oxp_7980b762-6ad3-435b-bf8d-91d18dcf75d4/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.294116179Z","hostname":"buskin","pid":1566,"name":"oxp_7980b762-6ad3-435b-bf8d-91d18dcf75d4/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.294131779Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7980b762-6ad3-435b-bf8d-91d18dcf75d4/crypt/debug\"), full_name: \"oxp_7980b762-6ad3-435b-bf8d-91d18dcf75d4/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.324087884Z","hostname":"buskin","pid":1566,"name":"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crypt/zone/oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.324122085Z","hostname":"buskin","pid":1566,"name":"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crypt/zone/oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.324126665Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/723d2041-12e1-48e9-aa34-7148357fe80c/crypt/zone/oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\"), full_name: \"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crypt/zone/oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.387694396Z","hostname":"buskin","pid":1566,"name":"oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9/crypt/zone/oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.387732637Z","hostname":"buskin","pid":1566,"name":"oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9/crypt/zone/oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.387737497Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/47f34c1f-e641-403f-804d-b2c26f2cb1f9/crypt/zone/oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\"), full_name: \"oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9/crypt/zone/oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.447918123Z","hostname":"buskin","pid":1566,"name":"oxp_28fb7d98-1494-4570-b25a-aa56857fbe19/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.447956803Z","hostname":"buskin","pid":1566,"name":"oxp_28fb7d98-1494-4570-b25a-aa56857fbe19/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.447961383Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_28fb7d98-1494-4570-b25a-aa56857fbe19/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.49466791Z","hostname":"buskin","pid":1566,"name":"oxp_736d5541-d5bd-460c-a5f5-441d6996aa03/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.494702431Z","hostname":"buskin","pid":1566,"name":"oxp_736d5541-d5bd-460c-a5f5-441d6996aa03/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.494706451Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_736d5541-d5bd-460c-a5f5-441d6996aa03/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.542487289Z","hostname":"buskin","pid":1566,"name":"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crypt/zone/oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.54252122Z","hostname":"buskin","pid":1566,"name":"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crypt/zone/oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.542525289Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/9764f33b-5a77-475e-83b5-777615774b6a/crypt/zone/oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\"), full_name: \"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crypt/zone/oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.600610165Z","hostname":"buskin","pid":1566,"name":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone/oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.600646926Z","hostname":"buskin","pid":1566,"name":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone/oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.600650946Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone/oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\"), full_name: \"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone/oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/198593b5-c3c6-4ef5-879a-52095a2ef511)","v":0,"name":"SledAgent","level":50,"time":"2025-01-16T23:21:45.613983412Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.659651009Z","hostname":"buskin","pid":1566,"name":"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crypt/zone/oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.65970831Z","hostname":"buskin","pid":1566,"name":"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crypt/zone/oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.65971355Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/723d2041-12e1-48e9-aa34-7148357fe80c/crypt/zone/oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5\"), full_name: \"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crypt/zone/oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.718348016Z","hostname":"buskin","pid":1566,"name":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.718388727Z","hostname":"buskin","pid":1566,"name":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.718393757Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.767354558Z","hostname":"buskin","pid":1566,"name":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone/oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.767395798Z","hostname":"buskin","pid":1566,"name":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone/oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.767399619Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone/oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\"), full_name: \"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone/oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.826117606Z","hostname":"buskin","pid":1566,"name":"oxp_736d5541-d5bd-460c-a5f5-441d6996aa03/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.826158717Z","hostname":"buskin","pid":1566,"name":"oxp_736d5541-d5bd-460c-a5f5-441d6996aa03/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.826169877Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/736d5541-d5bd-460c-a5f5-441d6996aa03/crypt/zone\"), full_name: \"oxp_736d5541-d5bd-460c-a5f5-441d6996aa03/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.855410079Z","hostname":"buskin","pid":1566,"name":"oxp_7980b762-6ad3-435b-bf8d-91d18dcf75d4/crypt/zone/oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.855445539Z","hostname":"buskin","pid":1566,"name":"oxp_7980b762-6ad3-435b-bf8d-91d18dcf75d4/crypt/zone/oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.85545013Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7980b762-6ad3-435b-bf8d-91d18dcf75d4/crypt/zone/oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\"), full_name: \"oxp_7980b762-6ad3-435b-bf8d-91d18dcf75d4/crypt/zone/oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.916475322Z","hostname":"buskin","pid":1566,"name":"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.916516002Z","hostname":"buskin","pid":1566,"name":"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.916520712Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.964898112Z","hostname":"buskin","pid":1566,"name":"oxp_65ae8c88-c7ba-40ff-af14-085d448fc3c0/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.964933432Z","hostname":"buskin","pid":1566,"name":"oxp_65ae8c88-c7ba-40ff-af14-085d448fc3c0/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.964938132Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/65ae8c88-c7ba-40ff-af14-085d448fc3c0/crypt/debug\"), full_name: \"oxp_65ae8c88-c7ba-40ff-af14-085d448fc3c0/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.993741825Z","hostname":"buskin","pid":1566,"name":"oxp_65ae8c88-c7ba-40ff-af14-085d448fc3c0/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.993778126Z","hostname":"buskin","pid":1566,"name":"oxp_65ae8c88-c7ba-40ff-af14-085d448fc3c0/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:45.993782795Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_65ae8c88-c7ba-40ff-af14-085d448fc3c0/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.04296259Z","hostname":"buskin","pid":1566,"name":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.043005391Z","hostname":"buskin","pid":1566,"name":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.043009851Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/external_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.091977101Z","hostname":"buskin","pid":1566,"name":"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crypt/zone/oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.092012992Z","hostname":"buskin","pid":1566,"name":"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crypt/zone/oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.092017432Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/723d2041-12e1-48e9-aa34-7148357fe80c/crypt/zone/oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), full_name: \"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crypt/zone/oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.1507082Z","hostname":"buskin","pid":1566,"name":"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crypt/zone/oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.15074311Z","hostname":"buskin","pid":1566,"name":"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crypt/zone/oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.1507475Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/9764f33b-5a77-475e-83b5-777615774b6a/crypt/zone/oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9\"), full_name: \"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crypt/zone/oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.212134979Z","hostname":"buskin","pid":1566,"name":"oxp_736d5541-d5bd-460c-a5f5-441d6996aa03/crypt/zone/oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.21217245Z","hostname":"buskin","pid":1566,"name":"oxp_736d5541-d5bd-460c-a5f5-441d6996aa03/crypt/zone/oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.21217681Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/736d5541-d5bd-460c-a5f5-441d6996aa03/crypt/zone/oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\"), full_name: \"oxp_736d5541-d5bd-460c-a5f5-441d6996aa03/crypt/zone/oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.271881007Z","hostname":"buskin","pid":1566,"name":"oxp_28fb7d98-1494-4570-b25a-aa56857fbe19/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.271915107Z","hostname":"buskin","pid":1566,"name":"oxp_28fb7d98-1494-4570-b25a-aa56857fbe19/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.271919047Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/28fb7d98-1494-4570-b25a-aa56857fbe19/crypt/debug\"), full_name: \"oxp_28fb7d98-1494-4570-b25a-aa56857fbe19/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.30067471Z","hostname":"buskin","pid":1566,"name":"oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.3007112Z","hostname":"buskin","pid":1566,"name":"oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.30071617Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.349691161Z","hostname":"buskin","pid":1566,"name":"oxp_28fb7d98-1494-4570-b25a-aa56857fbe19/crypt/zone/oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.349736212Z","hostname":"buskin","pid":1566,"name":"oxp_28fb7d98-1494-4570-b25a-aa56857fbe19/crypt/zone/oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.349740522Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/28fb7d98-1494-4570-b25a-aa56857fbe19/crypt/zone/oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\"), full_name: \"oxp_28fb7d98-1494-4570-b25a-aa56857fbe19/crypt/zone/oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.408699254Z","hostname":"buskin","pid":1566,"name":"oxp_28fb7d98-1494-4570-b25a-aa56857fbe19/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.408734705Z","hostname":"buskin","pid":1566,"name":"oxp_28fb7d98-1494-4570-b25a-aa56857fbe19/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.408738665Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/28fb7d98-1494-4570-b25a-aa56857fbe19/crypt/zone\"), full_name: \"oxp_28fb7d98-1494-4570-b25a-aa56857fbe19/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.437387645Z","hostname":"buskin","pid":1566,"name":"oxp_7980b762-6ad3-435b-bf8d-91d18dcf75d4/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.437421706Z","hostname":"buskin","pid":1566,"name":"oxp_7980b762-6ad3-435b-bf8d-91d18dcf75d4/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.437426296Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_7980b762-6ad3-435b-bf8d-91d18dcf75d4/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.487803534Z","hostname":"buskin","pid":1566,"name":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.487838014Z","hostname":"buskin","pid":1566,"name":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.487842024Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.537046559Z","hostname":"buskin","pid":1566,"name":"oxp_7980b762-6ad3-435b-bf8d-91d18dcf75d4/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.53708371Z","hostname":"buskin","pid":1566,"name":"oxp_7980b762-6ad3-435b-bf8d-91d18dcf75d4/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.53708865Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7980b762-6ad3-435b-bf8d-91d18dcf75d4/crypt/zone\"), full_name: \"oxp_7980b762-6ad3-435b-bf8d-91d18dcf75d4/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.566436454Z","hostname":"buskin","pid":1566,"name":"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crypt/zone/oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.566469734Z","hostname":"buskin","pid":1566,"name":"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crypt/zone/oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.566474354Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/723d2041-12e1-48e9-aa34-7148357fe80c/crypt/zone/oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\"), full_name: \"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crypt/zone/oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/198593b5-c3c6-4ef5-879a-52095a2ef511)","v":0,"name":"SledAgent","level":50,"time":"2025-01-16T23:21:46.616325212Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.628085528Z","hostname":"buskin","pid":1566,"name":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone/oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.628097368Z","hostname":"buskin","pid":1566,"name":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone/oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.628101768Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone/oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\"), full_name: \"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone/oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.688114751Z","hostname":"buskin","pid":1566,"name":"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crypt/zone/oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.688152902Z","hostname":"buskin","pid":1566,"name":"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crypt/zone/oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.688157242Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/9764f33b-5a77-475e-83b5-777615774b6a/crypt/zone/oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58\"), full_name: \"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crypt/zone/oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.747008272Z","hostname":"buskin","pid":1566,"name":"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crypt/zone/oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.747048322Z","hostname":"buskin","pid":1566,"name":"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crypt/zone/oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.747053343Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/9764f33b-5a77-475e-83b5-777615774b6a/crypt/zone/oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\"), full_name: \"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crypt/zone/oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.804872953Z","hostname":"buskin","pid":1566,"name":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/zone/oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.804912904Z","hostname":"buskin","pid":1566,"name":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/zone/oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.804917094Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/zone/oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\"), full_name: \"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/zone/oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.863433618Z","hostname":"buskin","pid":1566,"name":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.863472419Z","hostname":"buskin","pid":1566,"name":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.863476349Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/debug\"), full_name: \"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.892114439Z","hostname":"buskin","pid":1566,"name":"oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.892149339Z","hostname":"buskin","pid":1566,"name":"oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.892153429Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/47f34c1f-e641-403f-804d-b2c26f2cb1f9/crypt/zone\"), full_name: \"oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.921120066Z","hostname":"buskin","pid":1566,"name":"oxp_7980b762-6ad3-435b-bf8d-91d18dcf75d4/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.921151636Z","hostname":"buskin","pid":1566,"name":"oxp_7980b762-6ad3-435b-bf8d-91d18dcf75d4/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.921155326Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_7980b762-6ad3-435b-bf8d-91d18dcf75d4/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.96876964Z","hostname":"buskin","pid":1566,"name":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone/oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.968806971Z","hostname":"buskin","pid":1566,"name":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone/oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:46.968811681Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone/oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb\"), full_name: \"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone/oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.03228195Z","hostname":"buskin","pid":1566,"name":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/zone/oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.032319311Z","hostname":"buskin","pid":1566,"name":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/zone/oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.032323201Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/zone/oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\"), full_name: \"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/zone/oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.094454954Z","hostname":"buskin","pid":1566,"name":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.094490355Z","hostname":"buskin","pid":1566,"name":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.094494325Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/debug\"), full_name: \"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.12341741Z","hostname":"buskin","pid":1566,"name":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.123452931Z","hostname":"buskin","pid":1566,"name":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.123457551Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.17133313Z","hostname":"buskin","pid":1566,"name":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/zone/oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.171373821Z","hostname":"buskin","pid":1566,"name":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/zone/oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.171377931Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/zone/oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\"), full_name: \"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/zone/oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.231649899Z","hostname":"buskin","pid":1566,"name":"oxp_65ae8c88-c7ba-40ff-af14-085d448fc3c0/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.23169066Z","hostname":"buskin","pid":1566,"name":"oxp_65ae8c88-c7ba-40ff-af14-085d448fc3c0/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.23169549Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/65ae8c88-c7ba-40ff-af14-085d448fc3c0/crypt/zone\"), full_name: \"oxp_65ae8c88-c7ba-40ff-af14-085d448fc3c0/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.26087117Z","hostname":"buskin","pid":1566,"name":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.260913491Z","hostname":"buskin","pid":1566,"name":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.260918201Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/zone\"), full_name: \"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.291211183Z","hostname":"buskin","pid":1566,"name":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.291244524Z","hostname":"buskin","pid":1566,"name":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.291248274Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/external_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.341149162Z","hostname":"buskin","pid":1566,"name":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/zone/oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.341182673Z","hostname":"buskin","pid":1566,"name":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/zone/oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.341186733Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/zone/oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053\"), full_name: \"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/zone/oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.400500462Z","hostname":"buskin","pid":1566,"name":"oxp_736d5541-d5bd-460c-a5f5-441d6996aa03/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.400536852Z","hostname":"buskin","pid":1566,"name":"oxp_736d5541-d5bd-460c-a5f5-441d6996aa03/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.400540783Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/736d5541-d5bd-460c-a5f5-441d6996aa03/crypt/debug\"), full_name: \"oxp_736d5541-d5bd-460c-a5f5-441d6996aa03/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.430230603Z","hostname":"buskin","pid":1566,"name":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.430269494Z","hostname":"buskin","pid":1566,"name":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.430274234Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone\"), full_name: \"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.459118608Z","hostname":"buskin","pid":1566,"name":"oxp_65ae8c88-c7ba-40ff-af14-085d448fc3c0/crypt/zone/oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.459159929Z","hostname":"buskin","pid":1566,"name":"oxp_65ae8c88-c7ba-40ff-af14-085d448fc3c0/crypt/zone/oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.459164789Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/65ae8c88-c7ba-40ff-af14-085d448fc3c0/crypt/zone/oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), full_name: \"oxp_65ae8c88-c7ba-40ff-af14-085d448fc3c0/crypt/zone/oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.519856154Z","hostname":"buskin","pid":1566,"name":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/zone/oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.519896535Z","hostname":"buskin","pid":1566,"name":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/zone/oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.519901495Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/zone/oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba\"), full_name: \"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/zone/oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.579666823Z","hostname":"buskin","pid":1566,"name":"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.579707284Z","hostname":"buskin","pid":1566,"name":"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.579711964Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/9764f33b-5a77-475e-83b5-777615774b6a/crypt/zone\"), full_name: \"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.608905934Z","hostname":"buskin","pid":1566,"name":"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crypt/zone/oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.608938435Z","hostname":"buskin","pid":1566,"name":"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crypt/zone/oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.608953805Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/723d2041-12e1-48e9-aa34-7148357fe80c/crypt/zone/oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db\"), full_name: \"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crypt/zone/oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/198593b5-c3c6-4ef5-879a-52095a2ef511)","v":0,"name":"SledAgent","level":50,"time":"2025-01-16T23:21:47.61959158Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.669010639Z","hostname":"buskin","pid":1566,"name":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone/oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.66905082Z","hostname":"buskin","pid":1566,"name":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone/oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.66905508Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone/oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278\"), full_name: \"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone/oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.728456321Z","hostname":"buskin","pid":1566,"name":"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.728493571Z","hostname":"buskin","pid":1566,"name":"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.728497411Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.776912721Z","hostname":"buskin","pid":1566,"name":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.776951112Z","hostname":"buskin","pid":1566,"name":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.776955932Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.825049656Z","hostname":"buskin","pid":1566,"name":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/clickhouse","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.825088646Z","hostname":"buskin","pid":1566,"name":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/clickhouse","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.825093357Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/clickhouse\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.874149669Z","hostname":"buskin","pid":1566,"name":"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.874186779Z","hostname":"buskin","pid":1566,"name":"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.874191009Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_9764f33b-5a77-475e-83b5-777615774b6a/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.922847924Z","hostname":"buskin","pid":1566,"name":"oxp_65ae8c88-c7ba-40ff-af14-085d448fc3c0/crypt/zone/oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.922888585Z","hostname":"buskin","pid":1566,"name":"oxp_65ae8c88-c7ba-40ff-af14-085d448fc3c0/crypt/zone/oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.922893525Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/65ae8c88-c7ba-40ff-af14-085d448fc3c0/crypt/zone/oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\"), full_name: \"oxp_65ae8c88-c7ba-40ff-af14-085d448fc3c0/crypt/zone/oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.982686603Z","hostname":"buskin","pid":1566,"name":"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.982723473Z","hostname":"buskin","pid":1566,"name":"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:47.982739133Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/723d2041-12e1-48e9-aa34-7148357fe80c/crypt/zone\"), full_name: \"oxp_723d2041-12e1-48e9-aa34-7148357fe80c/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.010996266Z","hostname":"buskin","pid":1566,"name":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.011036127Z","hostname":"buskin","pid":1566,"name":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.011040887Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.05910856Z","hostname":"buskin","pid":1566,"name":"oxp_7980b762-6ad3-435b-bf8d-91d18dcf75d4/crypt/zone/oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.059149541Z","hostname":"buskin","pid":1566,"name":"oxp_7980b762-6ad3-435b-bf8d-91d18dcf75d4/crypt/zone/oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.059153741Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7980b762-6ad3-435b-bf8d-91d18dcf75d4/crypt/zone/oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67\"), full_name: \"oxp_7980b762-6ad3-435b-bf8d-91d18dcf75d4/crypt/zone/oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.118120944Z","hostname":"buskin","pid":1566,"name":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone/oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.118156714Z","hostname":"buskin","pid":1566,"name":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone/oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.118161714Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone/oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\"), full_name: \"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/zone/oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:48.128896371Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"19.145222701s"} {"msg":"Writing ledger to /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/.omicron-datasets.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.179086084Z","hostname":"buskin","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/.omicron-datasets.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.179435341Z","hostname":"buskin","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"datasets ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.179785448Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:861"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.179911Z","hostname":"buskin","pid":1566,"uri":"/datasets","method":"PUT","req_id":"75704b7c-02f5-4e51-8b00-c5ed8cb40384","remote_addr":"[fd00:1122:3344:101::1]:50430","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3221815,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.212270812Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.212411464Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:35045"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.212653109Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.212662859Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.221898727Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.221945088Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.230306028Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.230341979Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.238699319Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.23873077Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.23873631Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.238776721Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.2829716Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.28300784Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.283013971Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.333446249Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.337314593Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.34129806Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.354178247Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.358208885Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:48.362100519Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/198593b5-c3c6-4ef5-879a-52095a2ef511)","v":0,"name":"SledAgent","level":50,"time":"2025-01-16T23:21:48.622210745Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Profile for oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:49.941120775Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:49.941251738Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Booting oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:49.94139801Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:50.426592878Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:50.905907224Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:50.905963625Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:50.905976235Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"wait for service Some(\"oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba\") failed: Property not found. retry in 48.400718ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:50.914431077Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Booting oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:50.914473698Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:50.918673269Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba\") failed: Property not found. retry in 90.401715ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:50.973128924Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb\") failed: Property not found. retry in 49.029755ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:51.037888388Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9\") failed: Property not found. retry in 58.889208ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:51.045881201Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba\") failed: Property not found. retry in 239.927241ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:51.073628544Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:51.093099118Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4307"} {"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:51.093141099Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4317","gateway":"fd00:1122:3344:101::1"} {"msg":"wait for service Some(\"oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb\") failed: Property not found. retry in 62.460798ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:51.095659757Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Default route already exists","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:51.100343317Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4331","gateway":"fd00:1122:3344:101::1"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:51.100354258Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4396"} {"msg":"wait for service Some(\"oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9\") failed: Property not found. retry in 90.200418ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:51.381917905Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba\") failed: Property not found. retry in 451.908896ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:51.40300984Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb\") failed: Property not found. retry in 104.965159ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:51.420642749Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9\") failed: Property not found. retry in 180.024558ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:51.471123219Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-16T23:21:51.497890842Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12380209166418342036"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-16T23:21:51.497933853Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"18107571463403427358"} {"msg":"wait for service Some(\"oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb\") failed: Property not found. retry in 413.104122ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:51.556783033Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:51.638796839Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4462"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:51.63885193Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4489"} {"msg":"wait for service Some(\"oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9\") failed: Property not found. retry in 496.006118ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:51.663173247Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:51.682046959Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4502"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:51.68208446Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4348"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:51.742771696Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4390"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:51.742811566Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4469","rack_subnet":"fd00:1122:3344::"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:51.765484732Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4480"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:51.765527333Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4522"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:51.844636112Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4554"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:51.844688283Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4560"} {"msg":"wait for service Some(\"oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba\") failed: Property not found. retry in 625.036921ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:51.867221406Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:51.940037214Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4601"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:51.940088455Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:370"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:51.978317728Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:420"} {"msg":"wait for service Some(\"oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb\") failed: Property not found. retry in 1.055456265s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:51.980604883Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.011594408Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:717"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.01170404Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:762"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.012160589Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:765","uptime":"2"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.012177349Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:444","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: Some(\n None,\n ),\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n tx_eq: None,\n },\n },\n },\n}"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.049284642Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.103002083Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.103069615Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.103078075Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.103086565Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.103093325Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.103099755Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.103108145Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.103114745Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.103122596Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.103129366Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.103137346Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.103143716Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.103150236Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.103166176Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.103172877Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.103177617Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.103182207Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.103186757Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.103191207Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.103195817Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.103200387Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.103205057Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:52.103209477Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"wait for service Some(\"oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9\") failed: Property not found. retry in 786.324036ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:52.168382399Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-16T23:21:52.292035834Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15343543213510289921"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-16T23:21:52.292171056Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"7841044830787084053"} {"msg":"wait for service Some(\"oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba\") failed: Property not found. retry in 1.493246246s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:52.501885974Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9\") failed: Property not found. retry in 657.390522ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:52.966351024Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb\") failed: Property not found. retry in 1.0427589s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:53.046069025Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9\") failed: Property not found. retry in 1.457306671s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:53.633236212Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba\") failed: Property not found. retry in 1.445979279s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:54.003829809Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb\") failed: Property not found. retry in 1.00253543s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:54.096831925Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9\") failed: Property not found. retry in 1.338879027s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:55.101710653Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb\") failed: Property not found. retry in 759.942037ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:55.110743237Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba\") failed: Property not found. retry in 1.350079394s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:55.460342751Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb\") failed: Property not found. retry in 1.024882278s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:55.880638933Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:56.924083303Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_internal_dns_fd10c057-8c9f-4db2-98b5-bcf5e0d536ba"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:56.924154474Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_internal_dns_5c1ab32c-50b8-4564-bf29-e388b915cbdb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:56.924160334Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_internal_dns_fd7a4ec0-c746-4aff-9239-7c8992ff75d9"} {"msg":"Writing ledger to /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:56.924168424Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:56.924586252Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:56.924905439Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"9aef1d7d-c012-4351-86e8-2dd430fc68f5","remote_addr":"[fd00:1122:3344:101::1]:35045","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8712420,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:56.926725873Z","hostname":"buskin","pid":1566,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:658"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:56.978029848Z","hostname":"buskin","pid":1566,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:658"} {"msg":"failed to write DNS configuration (will retry in 114.356557ms)","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:57.008084245Z","hostname":"buskin","pid":1566,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:678","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:2::1]:5353/config): client error (Connect): tcp connect error: Connection refused (os error 146): Connection refused (os error 146)"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.142334974Z","hostname":"buskin","pid":1566,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:658"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.188565391Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:694"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.188578602Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.219045587Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.21971078Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.21971799Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.249365639Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.249491732Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:50872"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.249838268Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.249844899Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.260424892Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.260476693Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.268408705Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.268446166Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.276026711Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.276069222Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.276238955Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.276367408Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.286927281Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.31971754Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.320369023Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.320374723Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.350301028Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 180, 222]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.350322468Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1264","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.350809158Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.350815188Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.351143934Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.351534061Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.376221706Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:57.395924904Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:58.022339284Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:21:58.022501637Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db\") failed: Property not found. retry in 39.550415ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:58.209315115Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db\") failed: Property not found. retry in 75.511055ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:58.258036211Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db\") failed: Property not found. retry in 259.163605ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:58.343092904Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db\") failed: Property not found. retry in 351.68857ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:58.615623809Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db\") failed: Property not found. retry in 578.184329ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:58.975914757Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db\") failed: Property not found. retry in 828.781799ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:21:59.566829177Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db\") failed: Property not found. retry in 1.037180427s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:00.405604875Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db\") failed: Property not found. retry in 769.977944ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:01.452132554Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-16T23:22:01.499366461Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12380209166418342036"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-16T23:22:01.499503023Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"18107571463403427358"} {"msg":"wait for service Some(\"oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db\") failed: Property not found. retry in 1.050605614s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:02.231793417Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-16T23:22:02.293805378Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15343543213510289921"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-16T23:22:02.29392801Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"7841044830787084053"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:03.301801356Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db"} {"msg":"Writing ledger to /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:03.301871508Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:03.302259935Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:03.302590282Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"b62137ab-7a7c-4764-86fb-9c0acc919dbe","remote_addr":"[fd00:1122:3344:101::1]:50872","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6052933,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:03.304674752Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:735"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:03.334684638Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:03.334820491Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:36607"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2025-01-16T23:22:03.344306003Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3915"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:03.344429765Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"02d2e30e-ced1-4f40-8e0f-e0c33402a7e1","remote_addr":"[fd00:1122:3344:101::1]:36607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9445,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:03.34729059Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:03.520000067Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:03.520134Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:43197"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-16T23:22:06.466755009Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"7399087247714968952"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-16T23:22:06.831888162Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"18163733479285228256"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-16T23:22:06.925224165Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12274258656289525428"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2025-01-16T23:22:07.529780925Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3915"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:07.529866047Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"ee4c8b0d-2ff2-46ce-af5c-3a881f27fe54","remote_addr":"[fd00:1122:3344:101::1]:43197","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009668,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:07.531510288Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:07.652991561Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:07.653127434Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:46486"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_5bea1f62-6d19-4a05-904a-8ec00054b4db': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2025-01-16T23:22:07.661648108Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3915"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:07.66179162Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"dd1372f6-9daa-4b08-a8f9-34e7948434c1","remote_addr":"[fd00:1122:3344:101::1]:46486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8609,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:07.664701836Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:07.825213149Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:07.825327431Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:49088"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:07.833744533Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"de7ba436-07c4-4d8a-82a5-e266b1256006","remote_addr":"[fd00:1122:3344:101::1]:49088","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8362,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:07.836797642Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:07.836805252Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:07.993066882Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:07.993188064Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:57466"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:08.001351521Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"c1445d69-7fff-470e-ba35-1d3056574d04","remote_addr":"[fd00:1122:3344:101::1]:57466","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8070,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:08.004349609Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:08.004360969Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:08.314482615Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:08.314604187Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:58084"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:08.322114852Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"4621cd79-1097-4351-9859-d4980f04c936","remote_addr":"[fd00:1122:3344:101::1]:58084","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7449,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:08.325221121Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:08.325231362Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:08.684780247Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:08.684892499Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:64539"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:08.692968974Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"10096442-24d3-4363-addf-2a326c14c573","remote_addr":"[fd00:1122:3344:101::1]:64539","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8025,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:08.696076964Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:08.696103124Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:08.894127948Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:08.89423471Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:51883"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:08.90260838Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"74ff933e-9619-416b-80f3-5ae54c5778de","remote_addr":"[fd00:1122:3344:101::1]:51883","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8325,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:08.905982445Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:08.905989875Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.284668737Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.28479844Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:53182"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.29314185Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"b024b3c0-d1ce-4160-a95b-fa7f1fcd7dd5","remote_addr":"[fd00:1122:3344:101::1]:53182","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8278,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.294604988Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:09.294613358Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.303093062Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.303208424Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:42271"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.322821901Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"8d25adc7-b954-45d7-aaa7-d248492c959e","remote_addr":"[fd00:1122:3344:101::1]:42271","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":19492,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 1078944965, ip_addr: 64.79.100.197, stratum: 3, ref_time: 1737069728.3353877, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.327087493Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.327096633Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1392"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4a7f469e-713f-435a-a668-41edc7319053 (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: d2b1f660-7424-4780-8b78-032cf4cd5278 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 528bb1ba-b28e-4a21-a60d-70c953eccb58 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: 68c82739-2730-47fe-bbf9-3a5150b872f5 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.358971255Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.359129998Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:47378"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.359394343Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.359401083Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.370038058Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.370083628Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.379421968Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.379480109Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.388100194Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.388155105Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.396969155Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.397020516Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.405786494Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.405829185Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.415913899Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.415954119Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.425039274Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.425080895Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.435219439Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.43528168Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.435521355Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:09.435662168Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:10.002516748Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:10.002579569Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:10.00258777Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:10.00259487Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:10.00260362Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:10.035785377Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:10.04009536Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:10.044423993Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:10.049491501Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:10.053472377Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:10.057895882Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:10.062769046Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:10.067516397Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:10.071695867Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:10.075422389Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:13.69799223Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:13.702504357Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:13.731091438Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:13.731532667Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:13.928151713Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:13.928459829Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:13.934656678Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:13.934803241Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:13.942950428Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:13.943102881Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5\") failed: Property not found. retry in 41.251865ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:13.960623538Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67\") failed: Property not found. retry in 31.957801ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:13.969944107Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67\") failed: Property not found. retry in 101.921868ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:14.018693976Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5\") failed: Property not found. retry in 113.38978ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:14.028042426Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053\") failed: Property not found. retry in 67.432452ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:14.103506319Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58\") failed: Property not found. retry in 42.412239ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:14.111966242Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278\") failed: Property not found. retry in 73.154099ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:14.119555218Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67\") failed: Property not found. retry in 272.518973ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:14.130443528Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5\") failed: Property not found. retry in 274.09695ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:14.152237428Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58\") failed: Property not found. retry in 130.129346ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:14.165187997Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053\") failed: Property not found. retry in 68.668502ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:14.18093249Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278\") failed: Property not found. retry in 52.790866ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:14.201007217Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053\") failed: Property not found. retry in 112.328147ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:14.257417153Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278\") failed: Property not found. retry in 274.144991ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:14.265091811Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58\") failed: Property not found. retry in 240.308355ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:14.307823954Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053\") failed: Property not found. retry in 582.457068ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:14.382142555Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67\") failed: Property not found. retry in 426.140028ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:14.416278353Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5\") failed: Property not found. retry in 509.153227ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:14.442682881Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278\") failed: Property not found. retry in 480.238406ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:14.639242476Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58\") failed: Property not found. retry in 440.488611ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:14.658123585Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67\") failed: Property not found. retry in 849.818435ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:14.852656496Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5\") failed: Property not found. retry in 488.659137ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:14.963359428Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053\") failed: Property not found. retry in 984.624513ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:14.975067293Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58\") failed: Property not found. retry in 515.337156ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:15.109819608Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278\") failed: Property not found. retry in 412.787211ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:15.128461756Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5\") failed: Property not found. retry in 949.679902ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:15.544191422Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278\") failed: Property not found. retry in 535.194106ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:15.553493512Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58\") failed: Property not found. retry in 951.236242ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:15.634101484Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67\") failed: Property not found. retry in 1.212603877s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:15.710812581Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053\") failed: Property not found. retry in 1.261145641s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:15.9662437Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278\") failed: Property not found. retry in 1.336088455s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:16.096144962Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5\") failed: Property not found. retry in 576.584308ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:16.50051821Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58\") failed: Property not found. retry in 707.626244ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:16.590945771Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67\") failed: Property not found. retry in 918.314562ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:16.928644785Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5\") failed: Property not found. retry in 708.69695ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:17.084058238Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053\") failed: Property not found. retry in 819.651923ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:17.234740979Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58\") failed: Property not found. retry in 1.269311062s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:17.306544512Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278\") failed: Property not found. retry in 508.780922ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:17.437936283Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5\") failed: Property not found. retry in 790.799816ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:17.800157739Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67\") failed: Property not found. retry in 1.339021403s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:17.853473855Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278\") failed: Property not found. retry in 798.379126ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:17.95442943Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053\") failed: Property not found. retry in 584.521567ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:18.061128063Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58\") failed: Property not found. retry in 1.023487966s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:18.584235457Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5\") failed: Property not found. retry in 884.081346ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:18.600235955Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053\") failed: Property not found. retry in 1.364407496s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:18.656503249Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278\") failed: Property not found. retry in 996.629908ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:18.762614493Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67\") failed: Property not found. retry in 847.194596ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:19.198586838Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5\") failed: Property not found. retry in 878.055178ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:19.494316143Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58\") failed: Property not found. retry in 772.398558ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:19.613734563Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278\") failed: Property not found. retry in 531.894405ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:19.765427714Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053\") failed: Property not found. retry in 574.71096ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:20.025531542Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67\") failed: Property not found. retry in 1.261079476s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:20.0524161Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278\") failed: Property not found. retry in 837.908202ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:20.304138847Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5\") failed: Property not found. retry in 1.204624874s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:20.378511612Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58\") failed: Property not found. retry in 904.817573ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:20.3946406Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053\") failed: Property not found. retry in 1.138517287s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:20.607905026Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278\") failed: Property not found. retry in 523.859654ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:21.149720271Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58\") failed: Property not found. retry in 831.267656ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:21.307084061Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67\") failed: Property not found. retry in 883.28342ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:21.318713405Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5\") failed: Property not found. retry in 858.57201ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:21.591649852Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:21.646822994Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56175"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:21.647076485Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"af16c55b-f9c7-4be8-aaf9-44b21f1fea09","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56175","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278\") failed: Property not found. retry in 819.707081ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:21.681963503Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053\") failed: Property not found. retry in 1.276816019s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:21.753343405Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58\") failed: Property not found. retry in 694.96559ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:22.145842823Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67\") failed: Property not found. retry in 1.146501024s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:22.209478349Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:22.289723484Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"307.417455ms"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:23.376783438Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_cockroachdb_528bb1ba-b28e-4a21-a60d-70c953eccb58"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:23.376860259Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_cockroachdb_ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:23.37686713Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_cockroachdb_68c82739-2730-47fe-bbf9-3a5150b872f5"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:23.376874719Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_cockroachdb_d2b1f660-7424-4780-8b78-032cf4cd5278"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:23.37688195Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053"} {"msg":"Writing ledger to /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:23.37691831Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:23.377366044Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:23.377766737Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"bd844774-7d27-4e44-8adf-0e44c3bc2188","remote_addr":"[fd00:1122:3344:101::1]:47378","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":13627218,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:23.416290759Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:36021"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:23.447195004Z","hostname":"buskin","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"f7790890-e5dc-49fb-b7b7-f3c8d54333a2","remote_addr":"[fd00:1122:3344:101::1]:36021","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService9/omicron6 in oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService9/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":30801,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:23.447463939Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1158","retry_after":"89.27238ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f7790890-e5dc-49fb-b7b7-f3c8d54333a2\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:22:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f7790890-e5dc-49fb-b7b7-f3c8d54333a2\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:23.569821155Z","hostname":"buskin","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"9d868151-0c58-46fd-80f1-85f2ae47fa67","remote_addr":"[fd00:1122:3344:101::1]:36021","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService9/omicron6 in oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService9/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":31434,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:23.570005829Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1158","retry_after":"109.262869ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9d868151-0c58-46fd-80f1-85f2ae47fa67\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:22:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9d868151-0c58-46fd-80f1-85f2ae47fa67\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:23.709370693Z","hostname":"buskin","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"0c78175c-563b-4047-82bb-5a3640f46489","remote_addr":"[fd00:1122:3344:101::1]:36021","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService9/omicron6 in oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService9/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":28409,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:23.709560797Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1158","retry_after":"111.491244ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0c78175c-563b-4047-82bb-5a3640f46489\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:22:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0c78175c-563b-4047-82bb-5a3640f46489\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:23.849103768Z","hostname":"buskin","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"9828649f-75bd-4fb1-b11f-3a59542eb0ba","remote_addr":"[fd00:1122:3344:101::1]:36021","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService9/omicron6 in oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService9/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":26664,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:23.849302648Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1158","retry_after":"221.484064ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9828649f-75bd-4fb1-b11f-3a59542eb0ba\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:22:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9828649f-75bd-4fb1-b11f-3a59542eb0ba\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:24.09818854Z","hostname":"buskin","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"f950ff35-4740-45f3-a1fb-f34fb7840d95","remote_addr":"[fd00:1122:3344:101::1]:36021","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService9/omicron6 in oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService9/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":26275,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:24.098360033Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1158","retry_after":"146.214603ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f950ff35-4740-45f3-a1fb-f34fb7840d95\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:22:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f950ff35-4740-45f3-a1fb-f34fb7840d95\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:24.274004876Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3758"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:47.417408951Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3776"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:49.596163378Z","hostname":"buskin","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"7d40d964-7a64-425f-aac8-1d7bc1d14539","remote_addr":"[fd00:1122:3344:101::1]:36021","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_4a7f469e-713f-435a-a668-41edc7319053': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::3]:32221 --file /opt/oxide/cockroachdb/sql/dbinit.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\nCONFIGURE ZONE 1\nCREATE TYPE\nCREATE TABLE\nCREATE TABLE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE TABLE\nCREATE TABLE\nCREATE INDEX\nCREATE TYPE\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE INDEX\nCREATE TABLE\nCREATE INDEX\n stderr: ERROR: restart transaction: TransactionRetryWithProtoRefreshError: TransactionAbortedError(ABORT_REASON_ABORT_SPAN): \"sql txn\" meta={id=70d33435 key=/NamespaceTable/30/1/106/0/\"public\"/4/1 pri=0.01307973 epo=0 ts=1737069769.325275911,1 min=1737069768.910647966,0 seq=443} lock=true stat=ABORTED rts=1737069768.910647966,0 wto=false gul=1737069769.410647966,0\nSQLSTATE: 40001\nHINT: See: https://www.cockroachlabs.com/docs/v22.1/transaction-retry-error-reference.html#abort_reason_abort_span\nERROR: restart transaction: TransactionRetryWithProtoRefreshError: TransactionAbortedError(ABORT_REASON_ABORT_SPAN): \"sql txn\" meta={id=70d33435 key=/NamespaceTable/30/1/106/0/\"public\"/4/1 pri=0.01307973 epo=0 ts=1737069769.325275911,1 min=1737069768.910647966,0 seq=443} lock=true stat=ABORTED rts=1737069768.910647966,0 wto=false gul=1737069769.410647966,0\nSQLSTATE: 40001\nHINT: See: https://www.cockroachlabs.com/docs/v22.1/transaction-retry-error-reference.html#abort_reason_abort_span\nFailed running \"sql\"\n","latency_us":25419355,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:49.599778537Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1158","retry_after":"196.905488ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7d40d964-7a64-425f-aac8-1d7bc1d14539\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:22:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7d40d964-7a64-425f-aac8-1d7bc1d14539\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:49.604426827Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"601.389846ms"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:49.829158905Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3758"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:49.948110095Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3776"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.544931797Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3799"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.545170112Z","hostname":"buskin","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"78a79fdd-1a53-436e-8097-68a4372346e9","remote_addr":"[fd00:1122:3344:101::1]:36021","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7769721,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.546007458Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49136"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.546601779Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"1b5c7eea-d99d-4a75-bf64-3f43e7376c5a","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49136","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":259,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4a7f469e-713f-435a-a668-41edc7319053 (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: d2b1f660-7424-4780-8b78-032cf4cd5278 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 528bb1ba-b28e-4a21-a60d-70c953eccb58 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: 68c82739-2730-47fe-bbf9-3a5150b872f5 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: f4635fcc-61fc-4ff9-8203-b560522a972b (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: dc5f3810-93b6-44f1-815c-44411abf99ad (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: bda5150c-3889-413e-a774-9d40d8f8536d (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a365f9e-6c70-4a27-ba2b-6123ad53013e (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 227c6e26-7635-44f8-ab05-835d9cccc56f (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 58c7648a-2f27-407e-a7ac-2bf32aa3eb75 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d41ab55b-e1a6-4706-8fb9-30f250e379ca (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 861d2408-b2e2-416d-a5c8-444212cd8663 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 43273bd0-1810-4edd-be2e-b5a92667be8a (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 02ac6d47-6979-4b37-b0b4-656336a6af63 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 325a9c1d-813c-4693-8cf2-b4f93c573895 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: bb490289-6569-4f8c-868b-d3dd7bd03ac6 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: eb6ad2c3-d718-4b25-a9c1-d6170a395fb0 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a243b058-fd7d-4081-979b-227b010730e9 (service), filesystem_pool: Some(ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External } } } }, OmicronZoneConfig { id: 10fd3267-f358-4a92-b50e-b95e29dff885 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 77258649-3670-42da-8357-0d10bcdb4fa7 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb (service), filesystem_pool: Some(ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.586872235Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.58821513Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:39593"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.589183979Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.589194699Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.603836231Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.603887592Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.614963185Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.615011666Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.625661762Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.625702222Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.635242936Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.635282817Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.644713698Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.644753529Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.653801673Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.653840254Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.663713944Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.663758085Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.6728538Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.672893551Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.68267976Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.682725461Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.69441843Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.694457636Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.705235314Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.705274455Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.715736856Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.715787877Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.725204419Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.725244669Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.734710062Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.734748662Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.744800366Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.744837977Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.754056104Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.754098925Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.765290351Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.765329421Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.775030538Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.775070239Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.785683183Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.785733734Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.79589836Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.795946001Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.796210686Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.796343038Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:57.905639823Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:57.905699594Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:57.905708744Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:57.905714504Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:57.905721134Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:57.905736615Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:57.905744495Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:57.905749935Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:57.905755375Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:57.905762155Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:57.905767435Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:57.905770895Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:57.905776185Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:57.905780236Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:57.905784145Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:57.905789486Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:57.905794616Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:57.905800686Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:57.905806026Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:57.905868827Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"7f40e855-92e0-4cb3-80d5-55d3fda2b251","remote_addr":"[fd00:1122:3344:101::1]:39593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\", TimeNotSynchronized), (\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\", TimeNotSynchronized), (\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\", TimeNotSynchronized), (\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\", TimeNotSynchronized), (\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\", TimeNotSynchronized), (\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\", TimeNotSynchronized), (\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\", TimeNotSynchronized), (\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\", TimeNotSynchronized), (\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\", TimeNotSynchronized), (\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\", TimeNotSynchronized), (\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\", TimeNotSynchronized), (\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\", TimeNotSynchronized), (\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\", TimeNotSynchronized), (\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\", TimeNotSynchronized), (\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\", TimeNotSynchronized), (\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\", TimeNotSynchronized), (\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\", TimeNotSynchronized), (\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\", TimeNotSynchronized), (\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\", TimeNotSynchronized)]","latency_us":318422,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:57.906076861Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"128.580097ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7f40e855-92e0-4cb3-80d5-55d3fda2b251\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:22:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7f40e855-92e0-4cb3-80d5-55d3fda2b251\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4a7f469e-713f-435a-a668-41edc7319053 (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: d2b1f660-7424-4780-8b78-032cf4cd5278 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 528bb1ba-b28e-4a21-a60d-70c953eccb58 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: 68c82739-2730-47fe-bbf9-3a5150b872f5 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: f4635fcc-61fc-4ff9-8203-b560522a972b (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: dc5f3810-93b6-44f1-815c-44411abf99ad (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: bda5150c-3889-413e-a774-9d40d8f8536d (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a365f9e-6c70-4a27-ba2b-6123ad53013e (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 227c6e26-7635-44f8-ab05-835d9cccc56f (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 58c7648a-2f27-407e-a7ac-2bf32aa3eb75 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d41ab55b-e1a6-4706-8fb9-30f250e379ca (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 861d2408-b2e2-416d-a5c8-444212cd8663 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 43273bd0-1810-4edd-be2e-b5a92667be8a (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 02ac6d47-6979-4b37-b0b4-656336a6af63 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 325a9c1d-813c-4693-8cf2-b4f93c573895 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: bb490289-6569-4f8c-868b-d3dd7bd03ac6 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: eb6ad2c3-d718-4b25-a9c1-d6170a395fb0 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a243b058-fd7d-4081-979b-227b010730e9 (service), filesystem_pool: Some(ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External } } } }, OmicronZoneConfig { id: 10fd3267-f358-4a92-b50e-b95e29dff885 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 77258649-3670-42da-8357-0d10bcdb4fa7 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb (service), filesystem_pool: Some(ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.036028694Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.03637754Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.036386051Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.04776038Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.047812801Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.059537266Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.059579607Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.071558218Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.071598459Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.082271604Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.082309765Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.093571832Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.093609703Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.103935661Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.103977052Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.115119307Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.115155588Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.126048957Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.126086998Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.136464628Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.136499399Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.146132124Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.146168136Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.155352362Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.155403423Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.165877524Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.165912015Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.177209873Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.177247413Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.188175454Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.188212295Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.199768037Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.199815388Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.210608176Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.210657157Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.220223351Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.220261742Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.230830695Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.230865966Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.241042292Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.241079163Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.251356Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.251406692Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.251598165Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.251752088Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.356337962Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.356450584Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.356460504Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.356467635Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.356473365Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.356478385Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.356485405Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.356490945Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.356494765Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.356501385Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.356506516Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.356511716Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.356521446Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.356526976Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.356532246Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.356537246Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.356541056Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.356549726Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.356555056Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.356664179Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"f637aca3-2e1f-47dc-a546-4e4ff46990d1","remote_addr":"[fd00:1122:3344:101::1]:39593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\", TimeNotSynchronized), (\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\", TimeNotSynchronized), (\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\", TimeNotSynchronized), (\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\", TimeNotSynchronized), (\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\", TimeNotSynchronized), (\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\", TimeNotSynchronized), (\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\", TimeNotSynchronized), (\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\", TimeNotSynchronized), (\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\", TimeNotSynchronized), (\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\", TimeNotSynchronized), (\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\", TimeNotSynchronized), (\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\", TimeNotSynchronized), (\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\", TimeNotSynchronized), (\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\", TimeNotSynchronized), (\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\", TimeNotSynchronized), (\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\", TimeNotSynchronized), (\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\", TimeNotSynchronized), (\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\", TimeNotSynchronized), (\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\", TimeNotSynchronized)]","latency_us":321343,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.356808611Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"133.752902ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f637aca3-2e1f-47dc-a546-4e4ff46990d1\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:22:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f637aca3-2e1f-47dc-a546-4e4ff46990d1\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4a7f469e-713f-435a-a668-41edc7319053 (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: d2b1f660-7424-4780-8b78-032cf4cd5278 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 528bb1ba-b28e-4a21-a60d-70c953eccb58 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: 68c82739-2730-47fe-bbf9-3a5150b872f5 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: f4635fcc-61fc-4ff9-8203-b560522a972b (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: dc5f3810-93b6-44f1-815c-44411abf99ad (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: bda5150c-3889-413e-a774-9d40d8f8536d (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a365f9e-6c70-4a27-ba2b-6123ad53013e (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 227c6e26-7635-44f8-ab05-835d9cccc56f (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 58c7648a-2f27-407e-a7ac-2bf32aa3eb75 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d41ab55b-e1a6-4706-8fb9-30f250e379ca (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 861d2408-b2e2-416d-a5c8-444212cd8663 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 43273bd0-1810-4edd-be2e-b5a92667be8a (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 02ac6d47-6979-4b37-b0b4-656336a6af63 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 325a9c1d-813c-4693-8cf2-b4f93c573895 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: bb490289-6569-4f8c-868b-d3dd7bd03ac6 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: eb6ad2c3-d718-4b25-a9c1-d6170a395fb0 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a243b058-fd7d-4081-979b-227b010730e9 (service), filesystem_pool: Some(ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External } } } }, OmicronZoneConfig { id: 10fd3267-f358-4a92-b50e-b95e29dff885 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 77258649-3670-42da-8357-0d10bcdb4fa7 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb (service), filesystem_pool: Some(ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.491690159Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.491998045Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.492011235Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.504251761Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.504291731Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.515352155Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.515387335Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.527872416Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.527908266Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.53797508Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.538006261Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.54888635Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.548935201Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.559359452Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.559392313Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.571934464Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.571969965Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.582866785Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.582900415Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.59403745Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.59407052Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.603988802Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.604030752Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.614666667Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.614700848Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.625391374Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.625432314Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.636268943Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.636303454Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.646652343Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.646687814Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.657129475Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.657177116Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.668218188Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.668252389Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.679005056Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.679038737Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.689099721Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.689134391Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.698397829Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.6984372Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.70930986Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.709356271Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.709522294Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.709645395Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.814792921Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.814846282Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.814854132Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.814860182Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.814865492Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.814871083Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.814876443Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.814882813Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.814886693Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.814891953Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.814895393Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.814900213Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.814905333Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.814926414Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.814930324Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.814933814Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.814938694Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.814941884Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.814946524Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.814985685Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"a6f8b992-8db0-4e63-b95a-dd1881363f13","remote_addr":"[fd00:1122:3344:101::1]:39593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\", TimeNotSynchronized), (\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\", TimeNotSynchronized), (\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\", TimeNotSynchronized), (\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\", TimeNotSynchronized), (\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\", TimeNotSynchronized), (\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\", TimeNotSynchronized), (\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\", TimeNotSynchronized), (\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\", TimeNotSynchronized), (\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\", TimeNotSynchronized), (\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\", TimeNotSynchronized), (\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\", TimeNotSynchronized), (\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\", TimeNotSynchronized), (\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\", TimeNotSynchronized), (\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\", TimeNotSynchronized), (\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\", TimeNotSynchronized), (\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\", TimeNotSynchronized), (\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\", TimeNotSynchronized), (\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\", TimeNotSynchronized), (\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\", TimeNotSynchronized)]","latency_us":324047,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:58.815130868Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"181.49997ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a6f8b992-8db0-4e63-b95a-dd1881363f13\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:22:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a6f8b992-8db0-4e63-b95a-dd1881363f13\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4a7f469e-713f-435a-a668-41edc7319053 (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: d2b1f660-7424-4780-8b78-032cf4cd5278 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 528bb1ba-b28e-4a21-a60d-70c953eccb58 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: 68c82739-2730-47fe-bbf9-3a5150b872f5 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: f4635fcc-61fc-4ff9-8203-b560522a972b (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: dc5f3810-93b6-44f1-815c-44411abf99ad (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: bda5150c-3889-413e-a774-9d40d8f8536d (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a365f9e-6c70-4a27-ba2b-6123ad53013e (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 227c6e26-7635-44f8-ab05-835d9cccc56f (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 58c7648a-2f27-407e-a7ac-2bf32aa3eb75 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d41ab55b-e1a6-4706-8fb9-30f250e379ca (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 861d2408-b2e2-416d-a5c8-444212cd8663 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 43273bd0-1810-4edd-be2e-b5a92667be8a (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 02ac6d47-6979-4b37-b0b4-656336a6af63 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 325a9c1d-813c-4693-8cf2-b4f93c573895 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: bb490289-6569-4f8c-868b-d3dd7bd03ac6 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: eb6ad2c3-d718-4b25-a9c1-d6170a395fb0 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a243b058-fd7d-4081-979b-227b010730e9 (service), filesystem_pool: Some(ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External } } } }, OmicronZoneConfig { id: 10fd3267-f358-4a92-b50e-b95e29dff885 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 77258649-3670-42da-8357-0d10bcdb4fa7 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb (service), filesystem_pool: Some(ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.996885264Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.997173322Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:58.997183062Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.009242905Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.009287026Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.020844038Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.020889619Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.032113395Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.032154576Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.041964945Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.042012466Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.051951067Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.051993388Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.062388068Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.062428299Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.072808699Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.07285954Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.083675868Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.083716469Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.093982997Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.094034808Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.105216593Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.105258364Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.115341658Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.115380239Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.126645096Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.126688776Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.138197838Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.138241049Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.148977056Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.149025877Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.159532509Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.15957798Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.169795537Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.169838287Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.180180317Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.180225647Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.190888993Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.190931224Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.201866304Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.201924296Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.211551231Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.211608062Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.211777105Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.211887667Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.311392264Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.311450745Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.311456585Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.311461685Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.311466365Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.311471225Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.311474495Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.311477765Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.311481125Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.311485935Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.311499366Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.311503176Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.311506636Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.311509906Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.311513806Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.311519926Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.311524676Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.311527726Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.311532646Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.311537596Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"eb19c9aa-3d34-4b28-966b-e43558c7e5f4","remote_addr":"[fd00:1122:3344:101::1]:39593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\", TimeNotSynchronized), (\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\", TimeNotSynchronized), (\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\", TimeNotSynchronized), (\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\", TimeNotSynchronized), (\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\", TimeNotSynchronized), (\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\", TimeNotSynchronized), (\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\", TimeNotSynchronized), (\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\", TimeNotSynchronized), (\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\", TimeNotSynchronized), (\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\", TimeNotSynchronized), (\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\", TimeNotSynchronized), (\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\", TimeNotSynchronized), (\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\", TimeNotSynchronized), (\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\", TimeNotSynchronized), (\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\", TimeNotSynchronized), (\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\", TimeNotSynchronized), (\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\", TimeNotSynchronized), (\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\", TimeNotSynchronized), (\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\", TimeNotSynchronized)]","latency_us":315380,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.311697429Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"137.680645ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"eb19c9aa-3d34-4b28-966b-e43558c7e5f4\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:22:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"eb19c9aa-3d34-4b28-966b-e43558c7e5f4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4a7f469e-713f-435a-a668-41edc7319053 (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: d2b1f660-7424-4780-8b78-032cf4cd5278 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 528bb1ba-b28e-4a21-a60d-70c953eccb58 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: 68c82739-2730-47fe-bbf9-3a5150b872f5 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: f4635fcc-61fc-4ff9-8203-b560522a972b (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: dc5f3810-93b6-44f1-815c-44411abf99ad (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: bda5150c-3889-413e-a774-9d40d8f8536d (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a365f9e-6c70-4a27-ba2b-6123ad53013e (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 227c6e26-7635-44f8-ab05-835d9cccc56f (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 58c7648a-2f27-407e-a7ac-2bf32aa3eb75 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d41ab55b-e1a6-4706-8fb9-30f250e379ca (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 861d2408-b2e2-416d-a5c8-444212cd8663 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 43273bd0-1810-4edd-be2e-b5a92667be8a (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 02ac6d47-6979-4b37-b0b4-656336a6af63 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 325a9c1d-813c-4693-8cf2-b4f93c573895 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: bb490289-6569-4f8c-868b-d3dd7bd03ac6 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: eb6ad2c3-d718-4b25-a9c1-d6170a395fb0 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a243b058-fd7d-4081-979b-227b010730e9 (service), filesystem_pool: Some(ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External } } } }, OmicronZoneConfig { id: 10fd3267-f358-4a92-b50e-b95e29dff885 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 77258649-3670-42da-8357-0d10bcdb4fa7 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb (service), filesystem_pool: Some(ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.449585685Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.449888751Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.449898111Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.461378582Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.461411613Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.47268142Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.47271396Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.48362381Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.483668211Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.494715134Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.494745315Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.505106854Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.505137285Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.515154557Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.515188138Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.524944576Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.524973577Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.534943279Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.534978769Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.544748107Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.544777358Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.554325012Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.554355455Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.564404496Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.564433237Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.573998591Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.574026551Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.58428388Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.58433375Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.594023226Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.594053907Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.604175352Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.604207392Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.613655734Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.613689855Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.623491584Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.623522744Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.633717001Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.633756612Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.642821076Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.642856467Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.652495482Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.652540213Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.652704377Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.652815479Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.751335626Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.751389937Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.751398227Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.751403597Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.751409047Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.751416307Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.751437298Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.751442978Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.751449268Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.751456398Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.751462008Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.751468748Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.751474199Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.751479189Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.751482749Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.751487829Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.751494439Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.751497979Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.751501519Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.751597531Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"a1ce92fd-18e4-43ed-b4a5-3021c10e02a2","remote_addr":"[fd00:1122:3344:101::1]:39593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\", TimeNotSynchronized), (\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\", TimeNotSynchronized), (\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\", TimeNotSynchronized), (\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\", TimeNotSynchronized), (\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\", TimeNotSynchronized), (\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\", TimeNotSynchronized), (\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\", TimeNotSynchronized), (\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\", TimeNotSynchronized), (\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\", TimeNotSynchronized), (\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\", TimeNotSynchronized), (\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\", TimeNotSynchronized), (\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\", TimeNotSynchronized), (\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\", TimeNotSynchronized), (\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\", TimeNotSynchronized), (\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\", TimeNotSynchronized), (\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\", TimeNotSynchronized), (\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\", TimeNotSynchronized), (\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\", TimeNotSynchronized), (\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\", TimeNotSynchronized)]","latency_us":302686,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:22:59.751775344Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"114.241208ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a1ce92fd-18e4-43ed-b4a5-3021c10e02a2\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:22:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a1ce92fd-18e4-43ed-b4a5-3021c10e02a2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4a7f469e-713f-435a-a668-41edc7319053 (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: d2b1f660-7424-4780-8b78-032cf4cd5278 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 528bb1ba-b28e-4a21-a60d-70c953eccb58 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: 68c82739-2730-47fe-bbf9-3a5150b872f5 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: f4635fcc-61fc-4ff9-8203-b560522a972b (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: dc5f3810-93b6-44f1-815c-44411abf99ad (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: bda5150c-3889-413e-a774-9d40d8f8536d (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a365f9e-6c70-4a27-ba2b-6123ad53013e (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 227c6e26-7635-44f8-ab05-835d9cccc56f (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 58c7648a-2f27-407e-a7ac-2bf32aa3eb75 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d41ab55b-e1a6-4706-8fb9-30f250e379ca (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 861d2408-b2e2-416d-a5c8-444212cd8663 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 43273bd0-1810-4edd-be2e-b5a92667be8a (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 02ac6d47-6979-4b37-b0b4-656336a6af63 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 325a9c1d-813c-4693-8cf2-b4f93c573895 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: bb490289-6569-4f8c-868b-d3dd7bd03ac6 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: eb6ad2c3-d718-4b25-a9c1-d6170a395fb0 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a243b058-fd7d-4081-979b-227b010730e9 (service), filesystem_pool: Some(ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External } } } }, OmicronZoneConfig { id: 10fd3267-f358-4a92-b50e-b95e29dff885 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 77258649-3670-42da-8357-0d10bcdb4fa7 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb (service), filesystem_pool: Some(ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.866746588Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.867052353Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.867060664Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.878392502Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.878448203Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.889047147Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.889090968Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.900394976Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.900440331Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.910094453Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.910146714Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.920401481Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.920444882Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.930626038Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.930671969Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.940482788Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.940525728Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.950223395Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.950264216Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.959619056Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.959661717Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.969318533Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.969360214Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.980758213Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.980800114Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.991645423Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:22:59.991686534Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.001980842Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.002023933Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.010947935Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.010992185Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.020848275Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.020892346Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.0314796Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.031523711Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.040936012Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.040978803Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.050323353Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.050364974Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.060762894Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.060805125Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.070224236Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.070274807Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.070472411Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.070597073Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.170548818Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.17061853Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.17062314Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.17063718Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.17064246Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.17064771Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.17065272Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.17065636Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.17065968Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.17066291Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.17066758Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.170670751Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.170675861Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.170680693Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.170687828Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.170691043Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.170694137Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.170697315Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.170712531Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.170716094Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"0211d365-a39e-4a22-94df-f5f08cdfb7e6","remote_addr":"[fd00:1122:3344:101::1]:39593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\", TimeNotSynchronized), (\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\", TimeNotSynchronized), (\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\", TimeNotSynchronized), (\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\", TimeNotSynchronized), (\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\", TimeNotSynchronized), (\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\", TimeNotSynchronized), (\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\", TimeNotSynchronized), (\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\", TimeNotSynchronized), (\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\", TimeNotSynchronized), (\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\", TimeNotSynchronized), (\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\", TimeNotSynchronized), (\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\", TimeNotSynchronized), (\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\", TimeNotSynchronized), (\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\", TimeNotSynchronized), (\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\", TimeNotSynchronized), (\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\", TimeNotSynchronized), (\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\", TimeNotSynchronized), (\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\", TimeNotSynchronized), (\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\", TimeNotSynchronized)]","latency_us":304659,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.170866384Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"187.031945ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0211d365-a39e-4a22-94df-f5f08cdfb7e6\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:23:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0211d365-a39e-4a22-94df-f5f08cdfb7e6\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4a7f469e-713f-435a-a668-41edc7319053 (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: d2b1f660-7424-4780-8b78-032cf4cd5278 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 528bb1ba-b28e-4a21-a60d-70c953eccb58 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: 68c82739-2730-47fe-bbf9-3a5150b872f5 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: f4635fcc-61fc-4ff9-8203-b560522a972b (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: dc5f3810-93b6-44f1-815c-44411abf99ad (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: bda5150c-3889-413e-a774-9d40d8f8536d (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a365f9e-6c70-4a27-ba2b-6123ad53013e (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 227c6e26-7635-44f8-ab05-835d9cccc56f (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 58c7648a-2f27-407e-a7ac-2bf32aa3eb75 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d41ab55b-e1a6-4706-8fb9-30f250e379ca (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 861d2408-b2e2-416d-a5c8-444212cd8663 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 43273bd0-1810-4edd-be2e-b5a92667be8a (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 02ac6d47-6979-4b37-b0b4-656336a6af63 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 325a9c1d-813c-4693-8cf2-b4f93c573895 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: bb490289-6569-4f8c-868b-d3dd7bd03ac6 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: eb6ad2c3-d718-4b25-a9c1-d6170a395fb0 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a243b058-fd7d-4081-979b-227b010730e9 (service), filesystem_pool: Some(ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External } } } }, OmicronZoneConfig { id: 10fd3267-f358-4a92-b50e-b95e29dff885 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 77258649-3670-42da-8357-0d10bcdb4fa7 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb (service), filesystem_pool: Some(ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.357875916Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.358195829Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.358202363Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.370131352Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.370175868Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.380485061Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.380527892Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.389856132Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.389897372Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.400000877Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.400037968Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.409827626Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.409865627Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.419030015Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.419066244Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.429440754Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.429483405Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.439527618Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.439569039Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.449686374Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.449726834Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.459400561Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.459440012Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.469951834Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.469991245Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.479447037Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.479496808Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.489894348Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.489937739Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.499794756Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.49983415Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.509805371Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.509845642Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.519969577Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.520008898Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.529785756Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.529828277Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.539764248Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.539806199Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.549324663Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.549366303Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.558492269Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.558544Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.558714464Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.558826946Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.661675516Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.661737868Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.661742677Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.661747708Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.661751218Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.661754888Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.661760128Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.661765338Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.661769658Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.661774878Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.661779798Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.661783508Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.661787118Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.661790458Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.661795589Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.661808759Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.661812599Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.661815949Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.661819169Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:00.661838669Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"9f8430a7-d72a-40c1-bc63-e80ce75d2b39","remote_addr":"[fd00:1122:3344:101::1]:39593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\", TimeNotSynchronized), (\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\", TimeNotSynchronized), (\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\", TimeNotSynchronized), (\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\", TimeNotSynchronized), (\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\", TimeNotSynchronized), (\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\", TimeNotSynchronized), (\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\", TimeNotSynchronized), (\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\", TimeNotSynchronized), (\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\", TimeNotSynchronized), (\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\", TimeNotSynchronized), (\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\", TimeNotSynchronized), (\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\", TimeNotSynchronized), (\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\", TimeNotSynchronized), (\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\", TimeNotSynchronized), (\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\", TimeNotSynchronized), (\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\", TimeNotSynchronized), (\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\", TimeNotSynchronized), (\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\", TimeNotSynchronized), (\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\", TimeNotSynchronized)]","latency_us":304648,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:00.662019113Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"424.896039ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9f8430a7-d72a-40c1-bc63-e80ce75d2b39\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:22:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9f8430a7-d72a-40c1-bc63-e80ce75d2b39\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4a7f469e-713f-435a-a668-41edc7319053 (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: d2b1f660-7424-4780-8b78-032cf4cd5278 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 528bb1ba-b28e-4a21-a60d-70c953eccb58 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: 68c82739-2730-47fe-bbf9-3a5150b872f5 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: f4635fcc-61fc-4ff9-8203-b560522a972b (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: dc5f3810-93b6-44f1-815c-44411abf99ad (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: bda5150c-3889-413e-a774-9d40d8f8536d (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a365f9e-6c70-4a27-ba2b-6123ad53013e (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 227c6e26-7635-44f8-ab05-835d9cccc56f (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 58c7648a-2f27-407e-a7ac-2bf32aa3eb75 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d41ab55b-e1a6-4706-8fb9-30f250e379ca (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 861d2408-b2e2-416d-a5c8-444212cd8663 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 43273bd0-1810-4edd-be2e-b5a92667be8a (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 02ac6d47-6979-4b37-b0b4-656336a6af63 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 325a9c1d-813c-4693-8cf2-b4f93c573895 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: bb490289-6569-4f8c-868b-d3dd7bd03ac6 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: eb6ad2c3-d718-4b25-a9c1-d6170a395fb0 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a243b058-fd7d-4081-979b-227b010730e9 (service), filesystem_pool: Some(ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External } } } }, OmicronZoneConfig { id: 10fd3267-f358-4a92-b50e-b95e29dff885 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 77258649-3670-42da-8357-0d10bcdb4fa7 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb (service), filesystem_pool: Some(ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.087396524Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.08767645Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.087685Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.099107Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.099151731Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.11108471Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.111129752Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.121748936Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.121787056Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.131653937Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.131692747Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.141436825Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.141473906Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.150564321Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.150604271Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.159648736Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.159685016Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.168803732Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.168840313Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.178314565Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.178351116Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.187459831Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.187497252Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.196422884Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.196457674Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.207746502Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.207774952Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.218127992Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.218172792Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.22791434Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.227954911Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.237298831Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.237340412Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.247704661Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.247740292Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.257047061Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.257084242Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.266142736Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.266179907Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.275646009Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.27568262Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.284711674Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.284766175Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.284951769Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.285075191Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:01.381746673Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:01.381805454Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:01.381811654Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:01.381815774Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:01.381820884Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:01.381824554Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:01.381829594Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:01.381834564Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:01.381839255Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:01.381844004Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:01.381847304Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:01.381863815Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:01.381867355Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:01.381872095Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:01.381876715Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:01.381880015Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:01.381883325Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:01.381886425Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:01.381889736Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.381893256Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"4c6d2d2e-575d-4154-9c68-12aba3ba8e5d","remote_addr":"[fd00:1122:3344:101::1]:39593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\", TimeNotSynchronized), (\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\", TimeNotSynchronized), (\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\", TimeNotSynchronized), (\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\", TimeNotSynchronized), (\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\", TimeNotSynchronized), (\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\", TimeNotSynchronized), (\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\", TimeNotSynchronized), (\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\", TimeNotSynchronized), (\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\", TimeNotSynchronized), (\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\", TimeNotSynchronized), (\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\", TimeNotSynchronized), (\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\", TimeNotSynchronized), (\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\", TimeNotSynchronized), (\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\", TimeNotSynchronized), (\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\", TimeNotSynchronized), (\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\", TimeNotSynchronized), (\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\", TimeNotSynchronized), (\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\", TimeNotSynchronized), (\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\", TimeNotSynchronized)]","latency_us":295199,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:01.382068409Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"410.694699ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4c6d2d2e-575d-4154-9c68-12aba3ba8e5d\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:23:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4c6d2d2e-575d-4154-9c68-12aba3ba8e5d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4a7f469e-713f-435a-a668-41edc7319053 (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: d2b1f660-7424-4780-8b78-032cf4cd5278 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 528bb1ba-b28e-4a21-a60d-70c953eccb58 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: 68c82739-2730-47fe-bbf9-3a5150b872f5 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: f4635fcc-61fc-4ff9-8203-b560522a972b (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: dc5f3810-93b6-44f1-815c-44411abf99ad (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: bda5150c-3889-413e-a774-9d40d8f8536d (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a365f9e-6c70-4a27-ba2b-6123ad53013e (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 227c6e26-7635-44f8-ab05-835d9cccc56f (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 58c7648a-2f27-407e-a7ac-2bf32aa3eb75 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d41ab55b-e1a6-4706-8fb9-30f250e379ca (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 861d2408-b2e2-416d-a5c8-444212cd8663 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 43273bd0-1810-4edd-be2e-b5a92667be8a (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 02ac6d47-6979-4b37-b0b4-656336a6af63 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 325a9c1d-813c-4693-8cf2-b4f93c573895 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: bb490289-6569-4f8c-868b-d3dd7bd03ac6 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: eb6ad2c3-d718-4b25-a9c1-d6170a395fb0 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a243b058-fd7d-4081-979b-227b010730e9 (service), filesystem_pool: Some(ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External } } } }, OmicronZoneConfig { id: 10fd3267-f358-4a92-b50e-b95e29dff885 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 77258649-3670-42da-8357-0d10bcdb4fa7 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb (service), filesystem_pool: Some(ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.792504373Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.792783028Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.792792069Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.804436413Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.804480083Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.815263789Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.815300002Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.826302354Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.826338464Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.836136653Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.836172897Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.845854409Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.84588538Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.855615047Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.855649488Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.865778523Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.865812334Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.875514631Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.875546571Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.8853335Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.88536737Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.89519699Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.89523118Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.905126301Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.905159321Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.914676285Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.914707935Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.927862309Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.927897369Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.9378073Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.937845101Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.947910205Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.947944305Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.958492788Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.958526069Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.969447899Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.96947954Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.981850918Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.981889869Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.99337401Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:01.993407361Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.003991925Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.004026805Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.004208089Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.004322571Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.102263817Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.102322228Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.102327298Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.102332258Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.102335799Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.102340799Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.102345639Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.102350309Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.102364279Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.102367989Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.102373089Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.102376679Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.102381679Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.10238659Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.10239157Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.10239501Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.10239833Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.10240185Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.1024052Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.10240881Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"bb68ffa5-1ace-4054-a7e5-7a05d96eb657","remote_addr":"[fd00:1122:3344:101::1]:39593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\", TimeNotSynchronized), (\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\", TimeNotSynchronized), (\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\", TimeNotSynchronized), (\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\", TimeNotSynchronized), (\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\", TimeNotSynchronized), (\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\", TimeNotSynchronized), (\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\", TimeNotSynchronized), (\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\", TimeNotSynchronized), (\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\", TimeNotSynchronized), (\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\", TimeNotSynchronized), (\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\", TimeNotSynchronized), (\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\", TimeNotSynchronized), (\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\", TimeNotSynchronized), (\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\", TimeNotSynchronized), (\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\", TimeNotSynchronized), (\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\", TimeNotSynchronized), (\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\", TimeNotSynchronized), (\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\", TimeNotSynchronized), (\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\", TimeNotSynchronized)]","latency_us":310637,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.102561503Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"343.292113ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bb68ffa5-1ace-4054-a7e5-7a05d96eb657\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:23:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bb68ffa5-1ace-4054-a7e5-7a05d96eb657\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4a7f469e-713f-435a-a668-41edc7319053 (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: d2b1f660-7424-4780-8b78-032cf4cd5278 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 528bb1ba-b28e-4a21-a60d-70c953eccb58 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: 68c82739-2730-47fe-bbf9-3a5150b872f5 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: f4635fcc-61fc-4ff9-8203-b560522a972b (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: dc5f3810-93b6-44f1-815c-44411abf99ad (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: bda5150c-3889-413e-a774-9d40d8f8536d (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a365f9e-6c70-4a27-ba2b-6123ad53013e (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 227c6e26-7635-44f8-ab05-835d9cccc56f (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 58c7648a-2f27-407e-a7ac-2bf32aa3eb75 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d41ab55b-e1a6-4706-8fb9-30f250e379ca (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 861d2408-b2e2-416d-a5c8-444212cd8663 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 43273bd0-1810-4edd-be2e-b5a92667be8a (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 02ac6d47-6979-4b37-b0b4-656336a6af63 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 325a9c1d-813c-4693-8cf2-b4f93c573895 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: bb490289-6569-4f8c-868b-d3dd7bd03ac6 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: eb6ad2c3-d718-4b25-a9c1-d6170a395fb0 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a243b058-fd7d-4081-979b-227b010730e9 (service), filesystem_pool: Some(ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External } } } }, OmicronZoneConfig { id: 10fd3267-f358-4a92-b50e-b95e29dff885 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 77258649-3670-42da-8357-0d10bcdb4fa7 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb (service), filesystem_pool: Some(ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.44615652Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.446475166Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.446483086Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.457474288Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.457522035Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.468176174Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.468230575Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.477709258Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.477750448Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.48873991Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.488779301Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.497513669Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.49755268Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.507382541Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.50742083Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.517986303Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.518030934Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.52663535Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.52667449Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.535901568Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.535935629Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.549036291Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.549075102Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.558509323Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.558547604Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.56765333Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.56769209Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.577379107Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.577419218Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.587371509Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.58741333Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.596117797Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.596154159Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.605456098Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.605496868Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.618667602Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.618711623Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.628877539Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.628917599Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.638175188Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.638213609Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.647774262Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.647832084Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.647998584Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.648096729Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.741064739Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.7411199Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.74112686Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.741132951Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.741140401Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.741163051Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.741170361Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.741175801Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.741179771Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.741185501Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.741190841Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.741195782Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.741201652Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.741207302Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.741214302Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.741220012Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.741225602Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.741231002Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.741237753Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:02.741347155Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"42cb6c64-6e61-4d9d-a0f2-6b5edde2bdc0","remote_addr":"[fd00:1122:3344:101::1]:39593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\", TimeNotSynchronized), (\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\", TimeNotSynchronized), (\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\", TimeNotSynchronized), (\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\", TimeNotSynchronized), (\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\", TimeNotSynchronized), (\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\", TimeNotSynchronized), (\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\", TimeNotSynchronized), (\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\", TimeNotSynchronized), (\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\", TimeNotSynchronized), (\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\", TimeNotSynchronized), (\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\", TimeNotSynchronized), (\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\", TimeNotSynchronized), (\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\", TimeNotSynchronized), (\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\", TimeNotSynchronized), (\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\", TimeNotSynchronized), (\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\", TimeNotSynchronized), (\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\", TimeNotSynchronized), (\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\", TimeNotSynchronized), (\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\", TimeNotSynchronized)]","latency_us":295867,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:02.741898245Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"668.298103ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"42cb6c64-6e61-4d9d-a0f2-6b5edde2bdc0\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:23:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"42cb6c64-6e61-4d9d-a0f2-6b5edde2bdc0\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4a7f469e-713f-435a-a668-41edc7319053 (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: d2b1f660-7424-4780-8b78-032cf4cd5278 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 528bb1ba-b28e-4a21-a60d-70c953eccb58 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: 68c82739-2730-47fe-bbf9-3a5150b872f5 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: f4635fcc-61fc-4ff9-8203-b560522a972b (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: dc5f3810-93b6-44f1-815c-44411abf99ad (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: bda5150c-3889-413e-a774-9d40d8f8536d (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a365f9e-6c70-4a27-ba2b-6123ad53013e (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 227c6e26-7635-44f8-ab05-835d9cccc56f (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 58c7648a-2f27-407e-a7ac-2bf32aa3eb75 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d41ab55b-e1a6-4706-8fb9-30f250e379ca (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 861d2408-b2e2-416d-a5c8-444212cd8663 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 43273bd0-1810-4edd-be2e-b5a92667be8a (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 02ac6d47-6979-4b37-b0b4-656336a6af63 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 325a9c1d-813c-4693-8cf2-b4f93c573895 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: bb490289-6569-4f8c-868b-d3dd7bd03ac6 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: eb6ad2c3-d718-4b25-a9c1-d6170a395fb0 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a243b058-fd7d-4081-979b-227b010730e9 (service), filesystem_pool: Some(ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External } } } }, OmicronZoneConfig { id: 10fd3267-f358-4a92-b50e-b95e29dff885 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 77258649-3670-42da-8357-0d10bcdb4fa7 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb (service), filesystem_pool: Some(ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.409743926Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.410025951Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.410035301Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.421829449Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.421886662Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.432932782Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.432971883Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.443453445Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.443502196Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.453208903Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.453253494Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.462556493Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.462598584Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.472362432Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.472417523Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.483048827Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.483090718Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.49302208Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.49306714Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.502772998Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.502811028Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.512692518Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.512740699Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.523611599Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.52365669Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.533669922Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.533717343Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.544003741Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.544042942Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.554459633Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.554507394Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.564864993Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.564903344Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.574158942Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.574217273Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.58340846Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.583456466Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.592472325Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.592513876Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.601621061Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.601664572Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.610605684Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.610671285Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.610847139Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.610970631Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:03.708654352Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:03.708716064Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:03.708723704Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:03.708729204Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:03.708734314Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:03.708737974Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:03.708743074Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:03.708746584Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:03.708751594Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:03.708756364Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:03.708760884Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:03.708765834Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:03.708770734Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:03.708775665Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:03.708779125Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:03.708782555Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:03.708785755Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:03.708799605Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:03.708804375Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:03.708834416Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"d4e12451-3750-4e9f-9fd6-58d457862549","remote_addr":"[fd00:1122:3344:101::1]:39593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\", TimeNotSynchronized), (\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\", TimeNotSynchronized), (\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\", TimeNotSynchronized), (\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\", TimeNotSynchronized), (\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\", TimeNotSynchronized), (\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\", TimeNotSynchronized), (\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\", TimeNotSynchronized), (\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\", TimeNotSynchronized), (\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\", TimeNotSynchronized), (\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\", TimeNotSynchronized), (\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\", TimeNotSynchronized), (\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\", TimeNotSynchronized), (\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\", TimeNotSynchronized), (\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\", TimeNotSynchronized), (\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\", TimeNotSynchronized), (\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\", TimeNotSynchronized), (\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\", TimeNotSynchronized), (\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\", TimeNotSynchronized), (\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\", TimeNotSynchronized)]","latency_us":299800,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:03.708978809Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"544.776062ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d4e12451-3750-4e9f-9fd6-58d457862549\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:23:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d4e12451-3750-4e9f-9fd6-58d457862549\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4a7f469e-713f-435a-a668-41edc7319053 (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: d2b1f660-7424-4780-8b78-032cf4cd5278 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 528bb1ba-b28e-4a21-a60d-70c953eccb58 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: 68c82739-2730-47fe-bbf9-3a5150b872f5 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: f4635fcc-61fc-4ff9-8203-b560522a972b (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: dc5f3810-93b6-44f1-815c-44411abf99ad (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: bda5150c-3889-413e-a774-9d40d8f8536d (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a365f9e-6c70-4a27-ba2b-6123ad53013e (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 227c6e26-7635-44f8-ab05-835d9cccc56f (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 58c7648a-2f27-407e-a7ac-2bf32aa3eb75 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d41ab55b-e1a6-4706-8fb9-30f250e379ca (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 861d2408-b2e2-416d-a5c8-444212cd8663 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 43273bd0-1810-4edd-be2e-b5a92667be8a (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 02ac6d47-6979-4b37-b0b4-656336a6af63 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 325a9c1d-813c-4693-8cf2-b4f93c573895 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: bb490289-6569-4f8c-868b-d3dd7bd03ac6 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: eb6ad2c3-d718-4b25-a9c1-d6170a395fb0 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a243b058-fd7d-4081-979b-227b010730e9 (service), filesystem_pool: Some(ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External } } } }, OmicronZoneConfig { id: 10fd3267-f358-4a92-b50e-b95e29dff885 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 77258649-3670-42da-8357-0d10bcdb4fa7 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb (service), filesystem_pool: Some(ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.254026264Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.254296359Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.25431549Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.265987284Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.266034956Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.276943295Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.276985356Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.286885797Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.286924648Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.29689204Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.296935751Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.307207328Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.307259979Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.316832174Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.316873984Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.326158173Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.326197334Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.336716017Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.336759487Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.347559986Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.347601746Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.357239952Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.357289783Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.366438779Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.36647563Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.376657426Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.376699627Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.386500895Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.386535306Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.395873706Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.395912126Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.404955291Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.406846607Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.41479906Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.414813741Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.42417636Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.424218492Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.433141696Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.433179174Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.443128564Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.443164507Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.452603528Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.452651479Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.452803542Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.452913174Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:04.549755419Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:04.549819701Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:04.54982407Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:04.549829086Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:04.549832198Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:04.549835498Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:04.549840251Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:04.549845005Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:04.549849589Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:04.54985557Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:04.549859101Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:04.549863851Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:04.549867111Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:04.549878662Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:04.549885272Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:04.549890962Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:04.549895332Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:04.549899492Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:04.549903272Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:04.549909072Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"dc2b3211-6e20-4b4a-b0f6-dff0f5056877","remote_addr":"[fd00:1122:3344:101::1]:39593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\", TimeNotSynchronized), (\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\", TimeNotSynchronized), (\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\", TimeNotSynchronized), (\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\", TimeNotSynchronized), (\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\", TimeNotSynchronized), (\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\", TimeNotSynchronized), (\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\", TimeNotSynchronized), (\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\", TimeNotSynchronized), (\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\", TimeNotSynchronized), (\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\", TimeNotSynchronized), (\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\", TimeNotSynchronized), (\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\", TimeNotSynchronized), (\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\", TimeNotSynchronized), (\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\", TimeNotSynchronized), (\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\", TimeNotSynchronized), (\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\", TimeNotSynchronized), (\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\", TimeNotSynchronized), (\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\", TimeNotSynchronized), (\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\", TimeNotSynchronized)]","latency_us":296588,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:04.550069205Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"918.817906ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dc2b3211-6e20-4b4a-b0f6-dff0f5056877\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:23:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dc2b3211-6e20-4b4a-b0f6-dff0f5056877\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4a7f469e-713f-435a-a668-41edc7319053 (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: d2b1f660-7424-4780-8b78-032cf4cd5278 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 528bb1ba-b28e-4a21-a60d-70c953eccb58 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: 68c82739-2730-47fe-bbf9-3a5150b872f5 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: f4635fcc-61fc-4ff9-8203-b560522a972b (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: dc5f3810-93b6-44f1-815c-44411abf99ad (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: bda5150c-3889-413e-a774-9d40d8f8536d (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a365f9e-6c70-4a27-ba2b-6123ad53013e (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 227c6e26-7635-44f8-ab05-835d9cccc56f (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 58c7648a-2f27-407e-a7ac-2bf32aa3eb75 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d41ab55b-e1a6-4706-8fb9-30f250e379ca (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 861d2408-b2e2-416d-a5c8-444212cd8663 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 43273bd0-1810-4edd-be2e-b5a92667be8a (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 02ac6d47-6979-4b37-b0b4-656336a6af63 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 325a9c1d-813c-4693-8cf2-b4f93c573895 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: bb490289-6569-4f8c-868b-d3dd7bd03ac6 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: eb6ad2c3-d718-4b25-a9c1-d6170a395fb0 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a243b058-fd7d-4081-979b-227b010730e9 (service), filesystem_pool: Some(ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External } } } }, OmicronZoneConfig { id: 10fd3267-f358-4a92-b50e-b95e29dff885 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 77258649-3670-42da-8357-0d10bcdb4fa7 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb (service), filesystem_pool: Some(ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.467252237Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.467567423Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.467583444Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.479248208Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.479286849Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.489919234Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.489957285Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.499720743Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.499757123Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.509174435Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.509220445Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.518136187Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.518169818Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.527678481Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.527712402Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.536906569Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.536943399Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.546505154Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.546541644Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.556849643Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.556889474Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.565846496Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.565882947Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.576785427Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.576823667Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.587069085Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.587107526Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.597425034Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.597459985Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.606779775Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.606824005Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.616612264Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.616646645Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.625310831Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.625342522Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.63405306Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.63408457Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.643635614Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.643675135Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.65536166Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.655404981Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.665255041Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.665299591Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.665482665Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.665599287Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:05.757610579Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:05.75765754Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:05.75766334Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:05.75766701Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:05.75767184Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:05.7576768Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:05.75768049Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:05.75768527Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:05.757691641Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:05.757695151Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:05.757711511Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:05.757716801Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:05.757720411Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:05.757723871Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:05.757727221Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:05.757730422Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:05.757734931Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:05.757739892Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:05.757743232Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:05.757746462Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"6ca1593a-f9a0-4931-92ce-ee7c41d0a2d9","remote_addr":"[fd00:1122:3344:101::1]:39593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\", TimeNotSynchronized), (\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\", TimeNotSynchronized), (\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\", TimeNotSynchronized), (\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\", TimeNotSynchronized), (\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\", TimeNotSynchronized), (\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\", TimeNotSynchronized), (\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\", TimeNotSynchronized), (\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\", TimeNotSynchronized), (\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\", TimeNotSynchronized), (\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\", TimeNotSynchronized), (\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\", TimeNotSynchronized), (\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\", TimeNotSynchronized), (\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\", TimeNotSynchronized), (\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\", TimeNotSynchronized), (\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\", TimeNotSynchronized), (\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\", TimeNotSynchronized), (\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\", TimeNotSynchronized), (\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\", TimeNotSynchronized), (\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\", TimeNotSynchronized)]","latency_us":291147,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:05.757882994Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"917.679522ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6ca1593a-f9a0-4931-92ce-ee7c41d0a2d9\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:23:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6ca1593a-f9a0-4931-92ce-ee7c41d0a2d9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4a7f469e-713f-435a-a668-41edc7319053 (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: d2b1f660-7424-4780-8b78-032cf4cd5278 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 528bb1ba-b28e-4a21-a60d-70c953eccb58 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: 68c82739-2730-47fe-bbf9-3a5150b872f5 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: f4635fcc-61fc-4ff9-8203-b560522a972b (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: dc5f3810-93b6-44f1-815c-44411abf99ad (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: bda5150c-3889-413e-a774-9d40d8f8536d (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a365f9e-6c70-4a27-ba2b-6123ad53013e (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 227c6e26-7635-44f8-ab05-835d9cccc56f (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 58c7648a-2f27-407e-a7ac-2bf32aa3eb75 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d41ab55b-e1a6-4706-8fb9-30f250e379ca (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 861d2408-b2e2-416d-a5c8-444212cd8663 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 43273bd0-1810-4edd-be2e-b5a92667be8a (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 02ac6d47-6979-4b37-b0b4-656336a6af63 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 325a9c1d-813c-4693-8cf2-b4f93c573895 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: bb490289-6569-4f8c-868b-d3dd7bd03ac6 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: eb6ad2c3-d718-4b25-a9c1-d6170a395fb0 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a243b058-fd7d-4081-979b-227b010730e9 (service), filesystem_pool: Some(ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External } } } }, OmicronZoneConfig { id: 10fd3267-f358-4a92-b50e-b95e29dff885 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 77258649-3670-42da-8357-0d10bcdb4fa7 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb (service), filesystem_pool: Some(ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.673895504Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.67422128Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.6742287Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.686216651Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.686254522Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.696851896Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.696888336Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.706448141Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.706490072Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.715555276Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.715584787Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.726798703Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.726824773Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.736998639Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.73703221Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.745595465Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.745620375Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.755554756Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.755594837Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.764902596Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.764929617Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.774072773Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.774100594Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.782980785Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.783015095Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.792569599Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.79260497Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.801899649Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.80194002Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.810604606Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.810638877Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.819349115Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.819382195Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.828600612Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.828633523Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.837188718Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.837219518Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.847663559Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.8477072Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.856713784Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.856742724Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.866155535Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.866201516Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.86636087Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.866494122Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:06.961684075Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:06.961729226Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:06.961735366Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:06.961740736Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:06.961745746Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:06.961751096Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:06.961765027Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:06.961770587Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:06.961774267Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:06.961777747Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:06.961783047Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:06.961786507Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:06.961793157Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:06.961798577Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:06.961803468Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:06.961806777Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:06.961809988Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:06.961813288Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:06.961816738Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:06.961869109Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"8655a40c-c126-4b56-b9eb-599e322c349a","remote_addr":"[fd00:1122:3344:101::1]:39593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\", TimeNotSynchronized), (\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\", TimeNotSynchronized), (\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\", TimeNotSynchronized), (\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\", TimeNotSynchronized), (\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\", TimeNotSynchronized), (\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\", TimeNotSynchronized), (\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\", TimeNotSynchronized), (\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\", TimeNotSynchronized), (\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\", TimeNotSynchronized), (\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\", TimeNotSynchronized), (\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\", TimeNotSynchronized), (\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\", TimeNotSynchronized), (\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\", TimeNotSynchronized), (\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\", TimeNotSynchronized), (\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\", TimeNotSynchronized), (\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\", TimeNotSynchronized), (\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\", TimeNotSynchronized), (\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\", TimeNotSynchronized), (\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\", TimeNotSynchronized)]","latency_us":288618,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:06.962033692Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"932.832927ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8655a40c-c126-4b56-b9eb-599e322c349a\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:23:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8655a40c-c126-4b56-b9eb-599e322c349a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4a7f469e-713f-435a-a668-41edc7319053 (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: d2b1f660-7424-4780-8b78-032cf4cd5278 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 528bb1ba-b28e-4a21-a60d-70c953eccb58 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: 68c82739-2730-47fe-bbf9-3a5150b872f5 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: f4635fcc-61fc-4ff9-8203-b560522a972b (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: dc5f3810-93b6-44f1-815c-44411abf99ad (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: bda5150c-3889-413e-a774-9d40d8f8536d (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a365f9e-6c70-4a27-ba2b-6123ad53013e (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 227c6e26-7635-44f8-ab05-835d9cccc56f (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 58c7648a-2f27-407e-a7ac-2bf32aa3eb75 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d41ab55b-e1a6-4706-8fb9-30f250e379ca (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 861d2408-b2e2-416d-a5c8-444212cd8663 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 43273bd0-1810-4edd-be2e-b5a92667be8a (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 02ac6d47-6979-4b37-b0b4-656336a6af63 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 325a9c1d-813c-4693-8cf2-b4f93c573895 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: bb490289-6569-4f8c-868b-d3dd7bd03ac6 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: eb6ad2c3-d718-4b25-a9c1-d6170a395fb0 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a243b058-fd7d-4081-979b-227b010730e9 (service), filesystem_pool: Some(ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External } } } }, OmicronZoneConfig { id: 10fd3267-f358-4a92-b50e-b95e29dff885 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 77258649-3670-42da-8357-0d10bcdb4fa7 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb (service), filesystem_pool: Some(ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:07.893907687Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:07.894201793Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:07.894211193Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:07.906145393Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:07.906178794Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:07.915640176Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:07.915676176Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:07.92522533Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:07.925258281Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:07.934976248Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:07.935008679Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:07.944363769Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:07.94439597Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:07.953646288Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:07.953677048Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:07.962407606Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:07.962439347Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:07.971222986Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:07.971256357Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:07.9813Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:07.981333351Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:07.991767652Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:07.991801103Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:08.001846126Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:08.001879377Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:08.010853289Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:08.01088335Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:08.02021083Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:08.02024264Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:08.030495348Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:08.030530008Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:08.03943214Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:08.0394612Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:08.049165227Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:08.049195978Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:08.058596179Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:08.058626899Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:08.068088512Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:08.068120412Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:08.077434152Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:08.077464402Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:08.08626522Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:08.086297662Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:08.086459485Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:08.086571517Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:08.180050638Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:08.180106069Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:08.180113619Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:08.180134169Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:08.180140399Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:08.180146599Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:08.18015284Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:08.18015897Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:08.1801632Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:08.18016784Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:08.18017366Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:08.18017771Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:08.18018404Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:08.18018996Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:08.1801965Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:08.18020098Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:08.180204931Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:08.180209211Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:08.180218641Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:08.180224721Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"bf853891-27bc-4429-92a4-8b2acf06ef62","remote_addr":"[fd00:1122:3344:101::1]:39593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\", TimeNotSynchronized), (\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\", TimeNotSynchronized), (\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\", TimeNotSynchronized), (\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\", TimeNotSynchronized), (\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\", TimeNotSynchronized), (\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\", TimeNotSynchronized), (\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\", TimeNotSynchronized), (\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\", TimeNotSynchronized), (\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\", TimeNotSynchronized), (\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\", TimeNotSynchronized), (\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\", TimeNotSynchronized), (\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\", TimeNotSynchronized), (\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\", TimeNotSynchronized), (\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\", TimeNotSynchronized), (\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\", TimeNotSynchronized), (\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\", TimeNotSynchronized), (\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\", TimeNotSynchronized), (\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\", TimeNotSynchronized), (\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\", TimeNotSynchronized)]","latency_us":286968,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:08.180383204Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"879.461819ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bf853891-27bc-4429-92a4-8b2acf06ef62\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:23:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bf853891-27bc-4429-92a4-8b2acf06ef62\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4a7f469e-713f-435a-a668-41edc7319053 (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: d2b1f660-7424-4780-8b78-032cf4cd5278 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 528bb1ba-b28e-4a21-a60d-70c953eccb58 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: 68c82739-2730-47fe-bbf9-3a5150b872f5 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: f4635fcc-61fc-4ff9-8203-b560522a972b (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: dc5f3810-93b6-44f1-815c-44411abf99ad (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: bda5150c-3889-413e-a774-9d40d8f8536d (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a365f9e-6c70-4a27-ba2b-6123ad53013e (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 227c6e26-7635-44f8-ab05-835d9cccc56f (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 58c7648a-2f27-407e-a7ac-2bf32aa3eb75 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d41ab55b-e1a6-4706-8fb9-30f250e379ca (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 861d2408-b2e2-416d-a5c8-444212cd8663 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 43273bd0-1810-4edd-be2e-b5a92667be8a (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 02ac6d47-6979-4b37-b0b4-656336a6af63 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 325a9c1d-813c-4693-8cf2-b4f93c573895 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: bb490289-6569-4f8c-868b-d3dd7bd03ac6 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: eb6ad2c3-d718-4b25-a9c1-d6170a395fb0 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a243b058-fd7d-4081-979b-227b010730e9 (service), filesystem_pool: Some(ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External } } } }, OmicronZoneConfig { id: 10fd3267-f358-4a92-b50e-b95e29dff885 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 77258649-3670-42da-8357-0d10bcdb4fa7 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb (service), filesystem_pool: Some(ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.058684607Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.058968053Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.058975743Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.070368792Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.070412433Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.080512388Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.080552062Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.091351726Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.091396437Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.100923871Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.100964431Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.109808782Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.112333761Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.11906089Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.11907672Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.128902Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.12894658Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.137635988Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.140127616Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.147914176Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.147927506Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.157227255Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.157271606Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.166869421Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.166915532Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.176125639Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.17618325Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.185064221Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.185110382Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.19487218Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.194920921Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.20473958Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.207264079Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.213933367Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.213942617Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.223262187Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.223312348Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.232221999Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.23227395Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.241898545Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.241949477Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.252559991Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.252611122Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.252815306Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.252935558Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:09.349313764Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:09.349401926Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:09.349424106Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:09.349430787Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:09.349439176Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:09.349451337Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:09.349456437Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:09.349463267Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:09.349469957Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:09.349476897Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:09.349483477Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:09.349491948Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:09.349500298Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:09.349505468Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:09.349510188Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:09.349532218Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:09.349537858Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:09.349543049Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:09.349548539Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:09.349556839Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"ce0eda58-2417-4081-9353-5b8ad7b285e4","remote_addr":"[fd00:1122:3344:101::1]:39593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\", TimeNotSynchronized), (\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\", TimeNotSynchronized), (\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\", TimeNotSynchronized), (\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\", TimeNotSynchronized), (\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\", TimeNotSynchronized), (\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\", TimeNotSynchronized), (\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\", TimeNotSynchronized), (\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\", TimeNotSynchronized), (\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\", TimeNotSynchronized), (\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\", TimeNotSynchronized), (\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\", TimeNotSynchronized), (\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\", TimeNotSynchronized), (\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\", TimeNotSynchronized), (\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\", TimeNotSynchronized), (\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\", TimeNotSynchronized), (\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\", TimeNotSynchronized), (\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\", TimeNotSynchronized), (\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\", TimeNotSynchronized), (\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\", TimeNotSynchronized)]","latency_us":291561,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:09.349714832Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"1.50285421s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ce0eda58-2417-4081-9353-5b8ad7b285e4\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:23:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ce0eda58-2417-4081-9353-5b8ad7b285e4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4a7f469e-713f-435a-a668-41edc7319053 (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: d2b1f660-7424-4780-8b78-032cf4cd5278 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 528bb1ba-b28e-4a21-a60d-70c953eccb58 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: 68c82739-2730-47fe-bbf9-3a5150b872f5 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: f4635fcc-61fc-4ff9-8203-b560522a972b (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: dc5f3810-93b6-44f1-815c-44411abf99ad (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: bda5150c-3889-413e-a774-9d40d8f8536d (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a365f9e-6c70-4a27-ba2b-6123ad53013e (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 227c6e26-7635-44f8-ab05-835d9cccc56f (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 58c7648a-2f27-407e-a7ac-2bf32aa3eb75 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d41ab55b-e1a6-4706-8fb9-30f250e379ca (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 861d2408-b2e2-416d-a5c8-444212cd8663 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 43273bd0-1810-4edd-be2e-b5a92667be8a (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 02ac6d47-6979-4b37-b0b4-656336a6af63 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 325a9c1d-813c-4693-8cf2-b4f93c573895 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: bb490289-6569-4f8c-868b-d3dd7bd03ac6 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: eb6ad2c3-d718-4b25-a9c1-d6170a395fb0 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a243b058-fd7d-4081-979b-227b010730e9 (service), filesystem_pool: Some(ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External } } } }, OmicronZoneConfig { id: 10fd3267-f358-4a92-b50e-b95e29dff885 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 77258649-3670-42da-8357-0d10bcdb4fa7 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb (service), filesystem_pool: Some(ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.850003703Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.850269078Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.850279458Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.86125778Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.863843719Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.872299412Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.872326193Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.882607461Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.884973546Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.892147704Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.892156384Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.901529045Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.901598506Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.911450396Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.911495807Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.922130352Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.922190703Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.931180106Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.931228847Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.941177859Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.94123672Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.952734371Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.955184188Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.962262305Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.962277425Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.972027653Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.972085304Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.982040055Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.982087087Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.992495647Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:10.992540768Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:11.003289995Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:11.003343573Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:11.013011682Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:11.013063753Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:11.022520535Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:11.022575346Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:11.032012728Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:11.032061769Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:11.042377388Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:11.044807604Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:11.052197801Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:11.052252478Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:11.052512123Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:11.052637225Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:11.148796187Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:11.148878859Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:11.148888349Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:11.148895499Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:11.148900159Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:11.148906269Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:11.148913209Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:11.148919539Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:11.148926149Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:11.148935591Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:11.148939895Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:11.14895408Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:11.148958711Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:11.14896787Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:11.148984261Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:11.148990591Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:11.148995421Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:11.149000061Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:11.149009751Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:11.149014531Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"45a3d7a0-93ce-4164-aa52-452082dad5b9","remote_addr":"[fd00:1122:3344:101::1]:39593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\", TimeNotSynchronized), (\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\", TimeNotSynchronized), (\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\", TimeNotSynchronized), (\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\", TimeNotSynchronized), (\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\", TimeNotSynchronized), (\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\", TimeNotSynchronized), (\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\", TimeNotSynchronized), (\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\", TimeNotSynchronized), (\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\", TimeNotSynchronized), (\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\", TimeNotSynchronized), (\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\", TimeNotSynchronized), (\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\", TimeNotSynchronized), (\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\", TimeNotSynchronized), (\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\", TimeNotSynchronized), (\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\", TimeNotSynchronized), (\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\", TimeNotSynchronized), (\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\", TimeNotSynchronized), (\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\", TimeNotSynchronized), (\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\", TimeNotSynchronized)]","latency_us":299678,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:11.149161144Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"2.523805238s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"45a3d7a0-93ce-4164-aa52-452082dad5b9\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:23:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"45a3d7a0-93ce-4164-aa52-452082dad5b9\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:12.550805205Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"514.321143ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4a7f469e-713f-435a-a668-41edc7319053 (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: d2b1f660-7424-4780-8b78-032cf4cd5278 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 528bb1ba-b28e-4a21-a60d-70c953eccb58 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: 68c82739-2730-47fe-bbf9-3a5150b872f5 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: f4635fcc-61fc-4ff9-8203-b560522a972b (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: dc5f3810-93b6-44f1-815c-44411abf99ad (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: bda5150c-3889-413e-a774-9d40d8f8536d (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a365f9e-6c70-4a27-ba2b-6123ad53013e (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 227c6e26-7635-44f8-ab05-835d9cccc56f (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 58c7648a-2f27-407e-a7ac-2bf32aa3eb75 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d41ab55b-e1a6-4706-8fb9-30f250e379ca (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 861d2408-b2e2-416d-a5c8-444212cd8663 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 43273bd0-1810-4edd-be2e-b5a92667be8a (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 02ac6d47-6979-4b37-b0b4-656336a6af63 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 325a9c1d-813c-4693-8cf2-b4f93c573895 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: bb490289-6569-4f8c-868b-d3dd7bd03ac6 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: eb6ad2c3-d718-4b25-a9c1-d6170a395fb0 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a243b058-fd7d-4081-979b-227b010730e9 (service), filesystem_pool: Some(ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External } } } }, OmicronZoneConfig { id: 10fd3267-f358-4a92-b50e-b95e29dff885 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 77258649-3670-42da-8357-0d10bcdb4fa7 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb (service), filesystem_pool: Some(ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.666825037Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.667116593Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.667123971Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.679511321Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.679569162Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.691052824Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.691105585Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.700871513Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.700914213Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.711367895Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.711414896Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.720573932Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.720630113Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.729501054Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.729558865Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.740104548Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.740158099Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.750897351Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.750951507Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.760499271Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.760547042Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.770056665Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.770105666Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.779360574Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.779415235Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.789711894Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.789757104Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.800368689Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.80041849Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.810236768Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.810287159Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.819113699Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.81916785Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.828986019Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.82903989Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.839337938Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.83939619Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.848562876Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.848612147Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.858579219Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.858624954Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.869415088Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.869487629Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.869701013Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.869835056Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:13.966825073Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:13.966897305Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:13.966904125Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:13.966915245Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:13.966919335Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:13.966925245Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:13.966945346Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:13.966951176Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:13.966956426Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:13.966964576Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:13.966971276Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:13.966977116Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:13.966984156Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:13.966989727Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:13.966995247Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:13.967003987Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:13.967008197Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:13.967012137Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:13.967016967Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:13.967022547Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"e402b8d8-c6c9-4457-b4bf-7a693a0e172f","remote_addr":"[fd00:1122:3344:101::1]:39593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\", TimeNotSynchronized), (\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\", TimeNotSynchronized), (\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\", TimeNotSynchronized), (\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\", TimeNotSynchronized), (\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\", TimeNotSynchronized), (\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\", TimeNotSynchronized), (\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\", TimeNotSynchronized), (\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\", TimeNotSynchronized), (\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\", TimeNotSynchronized), (\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\", TimeNotSynchronized), (\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\", TimeNotSynchronized), (\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\", TimeNotSynchronized), (\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\", TimeNotSynchronized), (\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\", TimeNotSynchronized), (\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\", TimeNotSynchronized), (\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\", TimeNotSynchronized), (\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\", TimeNotSynchronized), (\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\", TimeNotSynchronized), (\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\", TimeNotSynchronized)]","latency_us":300856,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:13.96715637Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"1.14424263s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e402b8d8-c6c9-4457-b4bf-7a693a0e172f\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:23:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e402b8d8-c6c9-4457-b4bf-7a693a0e172f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4a7f469e-713f-435a-a668-41edc7319053 (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: d2b1f660-7424-4780-8b78-032cf4cd5278 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 528bb1ba-b28e-4a21-a60d-70c953eccb58 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: 68c82739-2730-47fe-bbf9-3a5150b872f5 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: f4635fcc-61fc-4ff9-8203-b560522a972b (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: dc5f3810-93b6-44f1-815c-44411abf99ad (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: bda5150c-3889-413e-a774-9d40d8f8536d (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a365f9e-6c70-4a27-ba2b-6123ad53013e (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 227c6e26-7635-44f8-ab05-835d9cccc56f (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 58c7648a-2f27-407e-a7ac-2bf32aa3eb75 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d41ab55b-e1a6-4706-8fb9-30f250e379ca (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 861d2408-b2e2-416d-a5c8-444212cd8663 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 43273bd0-1810-4edd-be2e-b5a92667be8a (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 02ac6d47-6979-4b37-b0b4-656336a6af63 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 325a9c1d-813c-4693-8cf2-b4f93c573895 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: bb490289-6569-4f8c-868b-d3dd7bd03ac6 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: eb6ad2c3-d718-4b25-a9c1-d6170a395fb0 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a243b058-fd7d-4081-979b-227b010730e9 (service), filesystem_pool: Some(ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External } } } }, OmicronZoneConfig { id: 10fd3267-f358-4a92-b50e-b95e29dff885 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 77258649-3670-42da-8357-0d10bcdb4fa7 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb (service), filesystem_pool: Some(ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.108816255Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.109146922Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.109155822Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.120672513Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.120738015Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.13244989Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.132500631Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.143114495Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.143166327Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.152849013Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.152887674Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.163239983Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.163291644Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.17244162Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.172484741Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.182558685Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.182607916Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.191948046Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.191996244Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.202267635Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.202316136Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.211976802Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.212022683Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.221185539Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.22123393Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.232722141Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.232767122Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.242391037Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.242440158Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.251746337Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.251798349Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.261011656Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.261059677Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.270947637Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.270989748Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.28045253Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.280502211Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.289873312Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.289929033Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.300239502Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.300291072Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.310173443Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.310236794Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.310418198Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.31054535Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:15.409860522Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:15.409925514Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:15.409935854Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:15.409956094Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:15.409961464Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:15.409968115Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:15.409976555Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:15.409984725Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:15.409991135Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:15.409999035Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:15.410006885Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:15.410020526Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:15.410027556Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:15.410032626Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:15.410039026Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:15.410047246Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:15.410054186Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:15.410058976Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:15.410082447Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:15.410100277Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"c39d12f9-db8f-4767-a2b9-a2d5df8cbe34","remote_addr":"[fd00:1122:3344:101::1]:39593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\", TimeNotSynchronized), (\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\", TimeNotSynchronized), (\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\", TimeNotSynchronized), (\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\", TimeNotSynchronized), (\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\", TimeNotSynchronized), (\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\", TimeNotSynchronized), (\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\", TimeNotSynchronized), (\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\", TimeNotSynchronized), (\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\", TimeNotSynchronized), (\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\", TimeNotSynchronized), (\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\", TimeNotSynchronized), (\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\", TimeNotSynchronized), (\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\", TimeNotSynchronized), (\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\", TimeNotSynchronized), (\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\", TimeNotSynchronized), (\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\", TimeNotSynchronized), (\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\", TimeNotSynchronized), (\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\", TimeNotSynchronized), (\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\", TimeNotSynchronized)]","latency_us":301985,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:15.41026001Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"1.405946967s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c39d12f9-db8f-4767-a2b9-a2d5df8cbe34\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:23:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c39d12f9-db8f-4767-a2b9-a2d5df8cbe34\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4a7f469e-713f-435a-a668-41edc7319053 (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: d2b1f660-7424-4780-8b78-032cf4cd5278 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 528bb1ba-b28e-4a21-a60d-70c953eccb58 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: 68c82739-2730-47fe-bbf9-3a5150b872f5 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: f4635fcc-61fc-4ff9-8203-b560522a972b (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: dc5f3810-93b6-44f1-815c-44411abf99ad (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: bda5150c-3889-413e-a774-9d40d8f8536d (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a365f9e-6c70-4a27-ba2b-6123ad53013e (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 227c6e26-7635-44f8-ab05-835d9cccc56f (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 58c7648a-2f27-407e-a7ac-2bf32aa3eb75 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d41ab55b-e1a6-4706-8fb9-30f250e379ca (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 861d2408-b2e2-416d-a5c8-444212cd8663 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 43273bd0-1810-4edd-be2e-b5a92667be8a (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 02ac6d47-6979-4b37-b0b4-656336a6af63 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 325a9c1d-813c-4693-8cf2-b4f93c573895 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: bb490289-6569-4f8c-868b-d3dd7bd03ac6 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: eb6ad2c3-d718-4b25-a9c1-d6170a395fb0 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a243b058-fd7d-4081-979b-227b010730e9 (service), filesystem_pool: Some(ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External } } } }, OmicronZoneConfig { id: 10fd3267-f358-4a92-b50e-b95e29dff885 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 77258649-3670-42da-8357-0d10bcdb4fa7 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb (service), filesystem_pool: Some(ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.81338565Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.813743857Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.813752827Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.826032693Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.826102735Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.837971933Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.838027194Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.848564737Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.848619138Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.858087461Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.858132791Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.867982011Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.868033702Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.878495934Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.878544775Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.888207651Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.888259732Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.899182662Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.899240283Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.910039881Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.910088222Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.920297749Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.92034534Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.929936684Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.929992895Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.939168742Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.939229003Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.950332607Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.950378688Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.961097674Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.961157746Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.97124233Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.97128211Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.981346574Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.981403265Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.991082092Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:16.991139532Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:17.000132816Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:17.000189147Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:17.009200001Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:17.009254282Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:17.018312646Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:17.018362907Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:17.018563171Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:17.018682273Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:17.116181991Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:17.116253972Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:17.116269683Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:17.116276873Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:17.116289043Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:17.116297423Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:17.116311363Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:17.116319824Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:17.116327924Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:17.116334554Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:17.116341034Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:17.116346364Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:17.116352784Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:17.116361175Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:17.116375615Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:17.116382325Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:17.116388265Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:17.116395065Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:17.116399685Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:17.116410315Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"b2796a95-814b-4062-89bd-4f40c13eaa9d","remote_addr":"[fd00:1122:3344:101::1]:39593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\", TimeNotSynchronized), (\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\", TimeNotSynchronized), (\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\", TimeNotSynchronized), (\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\", TimeNotSynchronized), (\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\", TimeNotSynchronized), (\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\", TimeNotSynchronized), (\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\", TimeNotSynchronized), (\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\", TimeNotSynchronized), (\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\", TimeNotSynchronized), (\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\", TimeNotSynchronized), (\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\", TimeNotSynchronized), (\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\", TimeNotSynchronized), (\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\", TimeNotSynchronized), (\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\", TimeNotSynchronized), (\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\", TimeNotSynchronized), (\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\", TimeNotSynchronized), (\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\", TimeNotSynchronized), (\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\", TimeNotSynchronized), (\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\", TimeNotSynchronized)]","latency_us":303705,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:17.116571439Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"2.351061092s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b2796a95-814b-4062-89bd-4f40c13eaa9d\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:23:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b2796a95-814b-4062-89bd-4f40c13eaa9d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4a7f469e-713f-435a-a668-41edc7319053 (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: d2b1f660-7424-4780-8b78-032cf4cd5278 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 528bb1ba-b28e-4a21-a60d-70c953eccb58 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: 68c82739-2730-47fe-bbf9-3a5150b872f5 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: f4635fcc-61fc-4ff9-8203-b560522a972b (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: dc5f3810-93b6-44f1-815c-44411abf99ad (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: bda5150c-3889-413e-a774-9d40d8f8536d (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a365f9e-6c70-4a27-ba2b-6123ad53013e (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 227c6e26-7635-44f8-ab05-835d9cccc56f (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 58c7648a-2f27-407e-a7ac-2bf32aa3eb75 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d41ab55b-e1a6-4706-8fb9-30f250e379ca (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 861d2408-b2e2-416d-a5c8-444212cd8663 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 43273bd0-1810-4edd-be2e-b5a92667be8a (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 02ac6d47-6979-4b37-b0b4-656336a6af63 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 325a9c1d-813c-4693-8cf2-b4f93c573895 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: bb490289-6569-4f8c-868b-d3dd7bd03ac6 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: eb6ad2c3-d718-4b25-a9c1-d6170a395fb0 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a243b058-fd7d-4081-979b-227b010730e9 (service), filesystem_pool: Some(ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External } } } }, OmicronZoneConfig { id: 10fd3267-f358-4a92-b50e-b95e29dff885 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 77258649-3670-42da-8357-0d10bcdb4fa7 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb (service), filesystem_pool: Some(ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.462745709Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.463025524Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.463033914Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.474779361Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.47735676Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.485011708Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.485025598Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.495832616Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.495896267Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.506689075Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.506739386Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.516470054Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.516527265Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.52612682Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.52617401Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.536645582Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.536686423Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.547485821Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.547523461Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.557685437Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.557732458Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.569958633Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.570018185Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.579532068Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.579584279Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.588041612Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.588097873Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.596713659Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.5967675Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.60559474Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.605653161Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.615271966Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.615319007Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.626184246Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.626240828Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.637123337Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.637188878Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.647333654Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.647388625Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.657053951Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.657110092Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.666891522Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.666941161Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.667182416Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.667328579Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:19.771465584Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:19.771537066Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:19.771546106Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:19.771554786Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:19.771559606Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:19.771566016Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:19.771570976Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:19.771583837Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:19.771590419Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:19.771597994Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:19.771603629Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:19.771621707Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:19.771628667Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:19.771634787Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:19.771646438Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:19.771651328Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:19.771655798Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:19.771660278Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:19.771668088Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:19.771673828Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"7b8a01b9-cc61-443b-8ae7-cd547e9aa162","remote_addr":"[fd00:1122:3344:101::1]:39593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\", TimeNotSynchronized), (\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\", TimeNotSynchronized), (\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\", TimeNotSynchronized), (\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\", TimeNotSynchronized), (\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\", TimeNotSynchronized), (\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\", TimeNotSynchronized), (\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\", TimeNotSynchronized), (\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\", TimeNotSynchronized), (\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\", TimeNotSynchronized), (\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\", TimeNotSynchronized), (\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\", TimeNotSynchronized), (\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\", TimeNotSynchronized), (\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\", TimeNotSynchronized), (\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\", TimeNotSynchronized), (\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\", TimeNotSynchronized), (\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\", TimeNotSynchronized), (\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\", TimeNotSynchronized), (\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\", TimeNotSynchronized), (\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\", TimeNotSynchronized)]","latency_us":309647,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:19.771855872Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"2.90839567s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7b8a01b9-cc61-443b-8ae7-cd547e9aa162\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:23:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7b8a01b9-cc61-443b-8ae7-cd547e9aa162\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4a7f469e-713f-435a-a668-41edc7319053 (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: d2b1f660-7424-4780-8b78-032cf4cd5278 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 528bb1ba-b28e-4a21-a60d-70c953eccb58 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: 68c82739-2730-47fe-bbf9-3a5150b872f5 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: f4635fcc-61fc-4ff9-8203-b560522a972b (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: dc5f3810-93b6-44f1-815c-44411abf99ad (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: bda5150c-3889-413e-a774-9d40d8f8536d (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a365f9e-6c70-4a27-ba2b-6123ad53013e (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 227c6e26-7635-44f8-ab05-835d9cccc56f (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 58c7648a-2f27-407e-a7ac-2bf32aa3eb75 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d41ab55b-e1a6-4706-8fb9-30f250e379ca (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 861d2408-b2e2-416d-a5c8-444212cd8663 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 43273bd0-1810-4edd-be2e-b5a92667be8a (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 02ac6d47-6979-4b37-b0b4-656336a6af63 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 325a9c1d-813c-4693-8cf2-b4f93c573895 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: bb490289-6569-4f8c-868b-d3dd7bd03ac6 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: eb6ad2c3-d718-4b25-a9c1-d6170a395fb0 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a243b058-fd7d-4081-979b-227b010730e9 (service), filesystem_pool: Some(ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External } } } }, OmicronZoneConfig { id: 10fd3267-f358-4a92-b50e-b95e29dff885 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 77258649-3670-42da-8357-0d10bcdb4fa7 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb (service), filesystem_pool: Some(ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.674114541Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.674401336Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.674409936Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.685387898Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.685444499Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.696892649Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.6969429Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.707191467Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.709703636Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.716566158Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.716578798Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.727153832Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.727210393Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.737678445Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.737726086Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.748641966Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.748690237Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.759502354Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.759561796Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.770308533Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.770360744Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.78103245Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.78107933Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.791189104Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.791240385Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.802750387Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.802811248Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.81483855Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.814899301Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.824865323Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.824908634Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.833458058Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.833503369Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.843443691Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.843489242Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.85433752Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.854389141Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.864918354Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.864973325Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.875509888Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.875571569Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.886781945Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.886834026Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.887000649Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.887120592Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:22.988721688Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:22.98880072Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:22.98881668Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:22.98882493Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:22.988837991Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:22.988844071Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:22.988848311Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:22.988852321Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:22.988871508Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:22.988878548Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:22.988883912Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:22.988889443Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:22.988893043Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:22.988896577Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:22.988906159Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:22.988911822Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:22.988915732Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:22.988921572Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:22.988925632Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:22.988958083Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"f188cbe5-949a-4b8c-9997-992ff9bae76f","remote_addr":"[fd00:1122:3344:101::1]:39593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\", TimeNotSynchronized), (\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\", TimeNotSynchronized), (\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\", TimeNotSynchronized), (\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\", TimeNotSynchronized), (\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\", TimeNotSynchronized), (\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\", TimeNotSynchronized), (\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\", TimeNotSynchronized), (\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\", TimeNotSynchronized), (\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\", TimeNotSynchronized), (\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\", TimeNotSynchronized), (\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\", TimeNotSynchronized), (\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\", TimeNotSynchronized), (\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\", TimeNotSynchronized), (\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\", TimeNotSynchronized), (\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\", TimeNotSynchronized), (\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\", TimeNotSynchronized), (\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\", TimeNotSynchronized), (\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\", TimeNotSynchronized), (\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\", TimeNotSynchronized)]","latency_us":315573,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:22.989133296Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"6.116230395s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f188cbe5-949a-4b8c-9997-992ff9bae76f\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:23:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f188cbe5-949a-4b8c-9997-992ff9bae76f\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:27.499471822Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65047"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:27.499560304Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"caa208c5-7ba6-445e-ba81-7d01665c8d5e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65047","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":30,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:28.060471655Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"2.055163345s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4a7f469e-713f-435a-a668-41edc7319053 (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: d2b1f660-7424-4780-8b78-032cf4cd5278 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 528bb1ba-b28e-4a21-a60d-70c953eccb58 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: 68c82739-2730-47fe-bbf9-3a5150b872f5 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: f4635fcc-61fc-4ff9-8203-b560522a972b (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: dc5f3810-93b6-44f1-815c-44411abf99ad (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: bda5150c-3889-413e-a774-9d40d8f8536d (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a365f9e-6c70-4a27-ba2b-6123ad53013e (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 227c6e26-7635-44f8-ab05-835d9cccc56f (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 58c7648a-2f27-407e-a7ac-2bf32aa3eb75 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d41ab55b-e1a6-4706-8fb9-30f250e379ca (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 861d2408-b2e2-416d-a5c8-444212cd8663 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 43273bd0-1810-4edd-be2e-b5a92667be8a (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 02ac6d47-6979-4b37-b0b4-656336a6af63 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 325a9c1d-813c-4693-8cf2-b4f93c573895 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: bb490289-6569-4f8c-868b-d3dd7bd03ac6 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: eb6ad2c3-d718-4b25-a9c1-d6170a395fb0 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a243b058-fd7d-4081-979b-227b010730e9 (service), filesystem_pool: Some(ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External } } } }, OmicronZoneConfig { id: 10fd3267-f358-4a92-b50e-b95e29dff885 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 77258649-3670-42da-8357-0d10bcdb4fa7 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb (service), filesystem_pool: Some(ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.090094743Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.090572012Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.090582062Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.102174826Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.102230056Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.11277646Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.112835361Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.122561028Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.122611229Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.131551891Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.131606472Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.142501642Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.142552563Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.151495535Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.151551526Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.160627941Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.160683472Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.169976471Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.172370307Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.179166798Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.179181778Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.188316125Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.188370426Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.19849737Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.198548951Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.207437253Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.207482203Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.216275583Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.216324604Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.225618183Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.225672564Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.234889271Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.234932782Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.243727501Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.243776073Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.252518561Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.252572292Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.2618032Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.26185285Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.270718761Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.270764052Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.279520751Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.279581262Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.279832028Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.27997658Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:29.374677943Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:29.374762775Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:29.374770955Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:29.374778825Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:29.374784565Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:29.374791645Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:29.374800306Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:29.374805826Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:29.374811516Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:29.374818576Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:29.374824156Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:29.374831106Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:29.374836476Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:29.374842946Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:29.374862147Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:29.374868017Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:29.374872267Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:29.374877727Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:29.374884777Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:29.374943368Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"8d841908-7182-4885-afd3-538ca56d9d9e","remote_addr":"[fd00:1122:3344:101::1]:39593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\", TimeNotSynchronized), (\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\", TimeNotSynchronized), (\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\", TimeNotSynchronized), (\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\", TimeNotSynchronized), (\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\", TimeNotSynchronized), (\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\", TimeNotSynchronized), (\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\", TimeNotSynchronized), (\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\", TimeNotSynchronized), (\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\", TimeNotSynchronized), (\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\", TimeNotSynchronized), (\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\", TimeNotSynchronized), (\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\", TimeNotSynchronized), (\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\", TimeNotSynchronized), (\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\", TimeNotSynchronized), (\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\", TimeNotSynchronized), (\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\", TimeNotSynchronized), (\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\", TimeNotSynchronized), (\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\", TimeNotSynchronized), (\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\", TimeNotSynchronized)]","latency_us":285441,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:29.375101011Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"7.128694319s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8d841908-7182-4885-afd3-538ca56d9d9e\", \"content-length\": \"124\", \"date\": \"Thu, 16 Jan 2025 23:23:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8d841908-7182-4885-afd3-538ca56d9d9e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fd10c057-8c9f-4db2-98b5-bcf5e0d536ba (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5c1ab32c-50b8-4564-bf29-e388b915cbdb (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fd7a4ec0-c746-4aff-9239-7c8992ff75d9 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 63b3013a-429b-4cc9-b01c-12c557b159b8, kind: Service { id: 5bea1f62-6d19-4a05-904a-8ec00054b4db }, name: Name(\"ntp-5bea1f62-6d19-4a05-904a-8ec00054b4db\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4a7f469e-713f-435a-a668-41edc7319053 (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: d2b1f660-7424-4780-8b78-032cf4cd5278 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 528bb1ba-b28e-4a21-a60d-70c953eccb58 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: 68c82739-2730-47fe-bbf9-3a5150b872f5 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: ddbdbd6d-fe49-44d5-a55e-3fd1e8e95e67 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: f4635fcc-61fc-4ff9-8203-b560522a972b (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: dc5f3810-93b6-44f1-815c-44411abf99ad (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: bda5150c-3889-413e-a774-9d40d8f8536d (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a365f9e-6c70-4a27-ba2b-6123ad53013e (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 227c6e26-7635-44f8-ab05-835d9cccc56f (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 58c7648a-2f27-407e-a7ac-2bf32aa3eb75 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d41ab55b-e1a6-4706-8fb9-30f250e379ca (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 861d2408-b2e2-416d-a5c8-444212cd8663 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 43273bd0-1810-4edd-be2e-b5a92667be8a (service), filesystem_pool: Some(ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External } } } }, OmicronZoneConfig { id: 02ac6d47-6979-4b37-b0b4-656336a6af63 (service), filesystem_pool: Some(ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 325a9c1d-813c-4693-8cf2-b4f93c573895 (service), filesystem_pool: Some(ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External } } } }, OmicronZoneConfig { id: bb490289-6569-4f8c-868b-d3dd7bd03ac6 (service), filesystem_pool: Some(ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External } } } }, OmicronZoneConfig { id: eb6ad2c3-d718-4b25-a9c1-d6170a395fb0 (service), filesystem_pool: Some(ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a243b058-fd7d-4081-979b-227b010730e9 (service), filesystem_pool: Some(ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External } } } }, OmicronZoneConfig { id: 10fd3267-f358-4a92-b50e-b95e29dff885 (service), filesystem_pool: Some(ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 77258649-3670-42da-8357-0d10bcdb4fa7 (service), filesystem_pool: Some(ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb (service), filesystem_pool: Some(ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.485469626Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.485768532Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.485776922Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.497385396Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.499805602Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.506904559Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.506928199Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.51684154Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.516893701Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.526269802Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.528678168Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.535713474Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.535729904Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.544499833Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.544549354Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.553513286Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.553564927Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.562543791Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.562597592Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.573502962Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.575894728Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.582914433Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.582923613Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.592395855Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.592448506Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.601788156Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.601836337Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.610856771Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.610905462Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.61965819Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.619713561Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.628694674Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.628742615Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.638396161Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.638454572Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.648618118Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.648675329Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.657224204Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.657268475Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.666237547Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.666286298Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.675610088Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.675661109Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.675860283Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.675975465Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.765520128Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.766959236Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.771929142Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.772037464Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.772052824Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.772065765Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.827783218Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.827858089Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.827878729Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:36.82788504Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.0392398Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.039432663Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.039442894Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.039472314Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.039480975Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.039525925Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.039535316Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.039541036Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.050772962Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.051474625Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.055991302Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.060316326Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.064923044Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.069942371Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.074811485Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.07974397Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.216649456Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.220967679Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.225342894Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.229417872Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.233604843Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.238311933Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.243260519Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Configuring new Omicron zone: oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.248211514Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.253297622Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.258541283Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.263682672Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.268410853Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.273522751Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.280590998Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.336762289Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Installing Omicron zone: oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.337182368Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.348470425Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.362341112Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.373595359Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.38822832Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.397415357Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.40637855Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.468284612Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.525673078Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.584784226Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.646164384Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.647706957Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.647732548Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.705053592Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 158, 198]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.705085022Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1264","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.707715923Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.707732564Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.776729001Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 208, 232]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.776767672Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1264","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.7782179Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.77823369Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.832620777Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 180, 6]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.832655997Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1264","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.83428933Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.83430962Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.896313044Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 215, 159]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.896345105Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1264","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.900136648Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.900157168Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.957129855Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 181, 71]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.957178486Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1264","zone_type":"external_dns"} {"msg":"no nat entry found for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.962594171Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.962618251Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.962719903Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.962731053Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.962808975Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.962819945Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.967151658Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:37.96876585Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"no nat entry found for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:38.03632662Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:38.036438283Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:38.036666947Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:38.040164925Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"creation of nat entry successful for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:38.040190105Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"no nat entry found for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:38.043657152Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:38.043673272Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:38.043765884Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"creation of nat entry successful for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:38.043902036Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:38.145071935Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:38.149310776Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:38.149350987Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:38.149598402Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:38.195803762Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:38.203792345Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:38.246302764Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Configuring new Omicron zone: oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:38.252550634Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:38.260323524Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:38.267487215Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Configuring new Omicron zone: oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:38.274260562Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:38.288011817Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:38.30528055Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:48.262868224Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"4.753129278s"} {"msg":"Profile for oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:49.361218135Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:49.362146032Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:49.362701293Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:49.402696173Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:49.409864871Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:49.410193538Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:49.450231739Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:49.450491044Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:49.458752883Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:49.459591379Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:49.460377664Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:49.460707551Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:49.461284082Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:49.46171836Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:49.462114478Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:49.462147943Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:49.469231205Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:49.476041706Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:49.482858807Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:49.488891933Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:49.495942879Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:49.501817842Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\") failed: Property not found. retry in 62.936576ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:49.812105437Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\") failed: Property not found. retry in 71.325975ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:49.82212194Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\") failed: Property not found. retry in 46.150595ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:49.831583282Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\") failed: Property not found. retry in 44.124499ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:49.839428683Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\") failed: Property not found. retry in 29.160665ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:49.848432656Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\") failed: Property not found. retry in 32.915508ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:49.856189456Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\") failed: Property not found. retry in 55.234308ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:49.864120928Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\") failed: Property not found. retry in 61.649787ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:49.872341427Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\") failed: Property not found. retry in 28.291315ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:49.893933713Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\") failed: Property not found. retry in 55.395764ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.002785169Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\") failed: Property not found. retry in 28.665202ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.017529743Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\") failed: Property not found. retry in 129.252333ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.03089309Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\") failed: Property not found. retry in 148.79774ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.043309859Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\") failed: Property not found. retry in 89.99886ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.053786421Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\") failed: Property not found. retry in 104.814141ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.065753331Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\") failed: Property not found. retry in 105.189105ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.077459117Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\") failed: Property not found. retry in 67.305515ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.08697753Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\") failed: Property not found. retry in 79.04995ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.094670388Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\") failed: Property not found. retry in 56.195435ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.101539741Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\") failed: Property not found. retry in 140.707838ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.109735858Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\") failed: Property not found. retry in 50.419051ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.126177305Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\") failed: Property not found. retry in 122.780137ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.250696503Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Profile for oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:50.250779395Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\") failed: Property not found. retry in 190.350152ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.57041096Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\") failed: Property not found. retry in 248.429786ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.613623132Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\") failed: Property not found. retry in 207.773722ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.647649217Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\") failed: Property not found. retry in 108.615323ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.682004389Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\") failed: Property not found. retry in 189.634196ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.706965329Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\") failed: Property not found. retry in 122.377784ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.73557827Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\") failed: Property not found. retry in 208.424242ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.762868016Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\") failed: Property not found. retry in 128.386669ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.788484239Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\") failed: Property not found. retry in 254.210746ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.813873698Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\") failed: Property not found. retry in 271.90928ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.830305644Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\") failed: Property not found. retry in 240.320144ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.847437538Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Booting oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:50.847545936Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\") failed: Property not found. retry in 228.578423ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.871921536Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\") failed: Property not found. retry in 556.570395ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.888841041Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\") failed: Property not found. retry in 516.192965ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.904277218Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\") failed: Property not found. retry in 524.896413ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.920475341Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\") failed: Property not found. retry in 591.038913ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.94068829Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\") failed: Property not found. retry in 565.924516ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.95730381Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\") failed: Property not found. retry in 551.99863ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.974753506Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\") failed: Property not found. retry in 515.306547ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:50.99052373Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\") failed: Property not found. retry in 470.763802ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:51.086905806Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Profile for oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:51.087906015Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\") failed: Property not found. retry in 399.086637ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:51.104580106Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Booting oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:51.104704047Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\") failed: Property not found. retry in 404.584579ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:51.130041746Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\") failed: Property not found. retry in 49.247961ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:51.145183088Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\") failed: Property not found. retry in 907.994911ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:51.163677744Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\") failed: Property not found. retry in 95.231603ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:51.209767682Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Profile for oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:51.252557996Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:51.252861382Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\") failed: Property not found. retry in 187.898286ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:51.321433572Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\") failed: Property not found. retry in 53.107609ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:51.433262156Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\") failed: Property not found. retry in 1.051009188s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:51.450240833Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\") failed: Property not found. retry in 720.306725ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:51.467903743Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\") failed: Property not found. retry in 568.62714ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:51.485996921Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\") failed: Property not found. retry in 60.24116ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:51.501895448Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\") failed: Property not found. retry in 843.304169ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:51.521798841Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\") failed: Property not found. retry in 797.323052ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:51.552638385Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\") failed: Property not found. retry in 496.107471ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:51.572578849Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\") failed: Property not found. retry in 904.788861ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:51.591470913Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\") failed: Property not found. retry in 979.933528ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:51.609343686Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\") failed: Property not found. retry in 846.437641ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:51.628324542Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\") failed: Property not found. retry in 907.782777ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:51.64688768Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\") failed: Property not found. retry in 1.170383903s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:51.666512688Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\") failed: Property not found. retry in 280.358558ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:51.680082469Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\") failed: Property not found. retry in 38.838157ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:51.695141546Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Profile for oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:51.695820672Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:51.696062887Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\") failed: Property not found. retry in 55.625746ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:51.843179659Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\") failed: Property not found. retry in 149.65266ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:51.927457282Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\") failed: Property not found. retry in 281.141163ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:51.983017932Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\") failed: Property not found. retry in 1.369005793s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:52.073892182Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\") failed: Property not found. retry in 551.041496ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:52.090018273Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\") failed: Property not found. retry in 693.903153ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:52.106239385Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\") failed: Property not found. retry in 465.15686ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:52.122231244Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\") failed: Property not found. retry in 68.606742ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:52.131849439Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\") failed: Property not found. retry in 1.283379569s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:52.205772922Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\") failed: Property not found. retry in 62.113076ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:52.219496877Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\") failed: Property not found. retry in 916.390202ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:52.277719448Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\") failed: Property not found. retry in 239.17036ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:52.295379018Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\") failed: Property not found. retry in 533.953268ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:52.36245439Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\") failed: Property not found. retry in 1.395489935s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:52.380755042Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\") failed: Property not found. retry in 781.424096ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:52.569761569Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\") failed: Property not found. retry in 713.404832ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:52.591387798Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\") failed: Property not found. retry in 978.036355ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:52.612974384Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\") failed: Property not found. retry in 721.240067ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:52.632022491Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\") failed: Property not found. retry in 584.615917ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:52.650247052Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\") failed: Property not found. retry in 1.048808554s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:52.666004385Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\") failed: Property not found. retry in 951.583529ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:52.683249967Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\") failed: Property not found. retry in 409.826196ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:52.700492049Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\") failed: Property not found. retry in 1.407443058s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:52.819421219Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\") failed: Property not found. retry in 1.283425656s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:52.852869343Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\") failed: Property not found. retry in 1.442396795s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:52.915610771Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\") failed: Property not found. retry in 646.769608ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.169644373Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Profile for oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:53.180496112Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:53.180539373Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:53.180579664Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:53.180859029Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:53.193223677Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:53.205750379Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\") failed: Property not found. retry in 1.072792305s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.24292635Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\") failed: Property not found. retry in 728.483879ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.261199137Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\") failed: Property not found. retry in 587.411059ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.316240497Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\") failed: Property not found. retry in 962.235554ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.362747622Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\") failed: Property not found. retry in 1.416534977s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.374442297Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\") failed: Property not found. retry in 28.666173ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.437854659Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\") failed: Property not found. retry in 30.035662ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.44887094Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\") failed: Property not found. retry in 689.715307ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.460502935Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\") failed: Property not found. retry in 35.161415ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.469378126Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\") failed: Property not found. retry in 130.696978ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.479165414Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\") failed: Property not found. retry in 144.312141ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.488688267Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\") failed: Property not found. retry in 1.1228528s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.497297093Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\") failed: Property not found. retry in 134.369009ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.512556877Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\") failed: Property not found. retry in 1.186077153s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.601595282Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\") failed: Property not found. retry in 140.909006ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.620806282Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\") failed: Property not found. retry in 1.126478328s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.64461907Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\") failed: Property not found. retry in 247.842513ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.657751563Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\") failed: Property not found. retry in 165.389846ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.671389026Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\") failed: Property not found. retry in 649.851327ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.724625664Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\") failed: Property not found. retry in 481.427841ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.775406048Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\") failed: Property not found. retry in 641.884167ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.787866078Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\") failed: Property not found. retry in 968.508254ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.826789468Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\") failed: Property not found. retry in 581.573707ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.848066328Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\") failed: Property not found. retry in 832.321009ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.91465358Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\") failed: Property not found. retry in 280.37253ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:53.924899967Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\") failed: Property not found. retry in 814.861559ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:54.22788827Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\") failed: Property not found. retry in 751.517762ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:54.247217862Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\") failed: Property not found. retry in 1.282771908s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:54.260706601Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\") failed: Property not found. retry in 668.234407ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:54.275023416Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\") failed: Property not found. retry in 1.148296261s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:54.287484095Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\") failed: Property not found. retry in 912.347503ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:54.300560326Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\") failed: Property not found. retry in 779.566325ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:54.326482484Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\") failed: Property not found. retry in 1.222608792s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:54.337413984Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\") failed: Property not found. retry in 1.167225468s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:54.367767487Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\") failed: Property not found. retry in 1.182032145s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:54.387679389Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\") failed: Property not found. retry in 602.530502ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:54.441601785Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\") failed: Property not found. retry in 818.640362ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:54.45280325Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\") failed: Property not found. retry in 1.394044551s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:54.63293613Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\") failed: Property not found. retry in 1.280979707s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:54.759628362Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\") failed: Property not found. retry in 780.04124ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:54.781947361Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\") failed: Property not found. retry in 1.485825651s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:54.799011649Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\") failed: Property not found. retry in 1.154251126s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:54.809602692Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\") failed: Property not found. retry in 1.409906561s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:54.820479431Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\") failed: Property not found. retry in 747.673952ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:54.958196996Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\") failed: Property not found. retry in 527.088512ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:55.013404896Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\") failed: Property not found. retry in 1.34026922s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:55.056277559Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\") failed: Property not found. retry in 1.306985788s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:55.068158788Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\") failed: Property not found. retry in 514.258725ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:55.121435471Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\") failed: Property not found. retry in 574.104536ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:55.228229512Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\") failed: Property not found. retry in 1.250929189s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:55.286612753Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\") failed: Property not found. retry in 761.598748ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:55.448826229Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\") failed: Property not found. retry in 609.834929ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:55.548490293Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\") failed: Property not found. retry in 1.117587456s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:55.560156537Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\") failed: Property not found. retry in 829.026294ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:55.570372343Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\") failed: Property not found. retry in 935.3255ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:55.582047127Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\") failed: Property not found. retry in 937.185648ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:55.592744033Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\") failed: Property not found. retry in 795.902383ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:55.601794636Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\") failed: Property not found. retry in 756.046323ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:55.64833989Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\") failed: Property not found. retry in 877.120668ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:55.71813438Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\") failed: Property not found. retry in 1.278686156s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:55.815662353Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\") failed: Property not found. retry in 860.903841ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:55.977740356Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\") failed: Property not found. retry in 1.355957541s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:56.04202512Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\") failed: Property not found. retry in 813.987369ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:56.054784866Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\") failed: Property not found. retry in 622.357382ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:56.173172719Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\") failed: Property not found. retry in 1.464214535s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:56.223916724Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\") failed: Property not found. retry in 1.282742974s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:56.244752534Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\") failed: Property not found. retry in 1.091327916s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:56.296809054Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\") failed: Property not found. retry in 1.017165583s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:56.389039125Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\") failed: Property not found. retry in 1.139468439s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:56.410540338Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\") failed: Property not found. retry in 1.480369151s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:56.421985588Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\") failed: Property not found. retry in 677.381355ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:56.433249234Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\") failed: Property not found. retry in 595.489524ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:56.44242143Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\") failed: Property not found. retry in 936.182407ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:56.526320551Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\") failed: Property not found. retry in 866.897802ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:56.538497816Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\") failed: Property not found. retry in 992.906845ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:56.545927608Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\") failed: Property not found. retry in 1.077343175s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:56.603506864Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\") failed: Property not found. retry in 1.431677463s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:56.686056689Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\") failed: Property not found. retry in 1.338469598s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:56.805961932Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\") failed: Property not found. retry in 1.102412412s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:56.847646842Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\") failed: Property not found. retry in 538.403129ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:56.93085507Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\") failed: Property not found. retry in 1.044214594s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:57.046722099Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\") failed: Property not found. retry in 943.471431ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:57.108501962Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\") failed: Property not found. retry in 1.349533611s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:57.181131407Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\") failed: Property not found. retry in 1.114731685s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:57.39682663Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\") failed: Property not found. retry in 989.862369ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:57.407355992Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\") failed: Property not found. retry in 1.37446876s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:57.415273394Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\") failed: Property not found. retry in 544.793681ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:57.423338589Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\") failed: Property not found. retry in 765.331642ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:57.471949302Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\") failed: Property not found. retry in 1.296232656s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:57.480318713Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\") failed: Property not found. retry in 1.002617947s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:57.538123593Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\") failed: Property not found. retry in 1.090452156s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:57.547932932Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\") failed: Property not found. retry in 560.763693ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:57.561051694Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\") failed: Property not found. retry in 726.266084ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:57.691162422Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\") failed: Property not found. retry in 1.435210762s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:57.700020982Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\") failed: Property not found. retry in 1.496759789s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:57.912407251Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\") failed: Property not found. retry in 1.29367522s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:57.959651608Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\") failed: Property not found. retry in 1.492179526s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:57.976744977Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\") failed: Property not found. retry in 1.151183924s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:58.061383572Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\") failed: Property not found. retry in 1.457101069s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:58.10086357Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\") failed: Property not found. retry in 1.31196763s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:58.126828749Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\") failed: Property not found. retry in 650.826305ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:58.135464965Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\") failed: Property not found. retry in 909.296197ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:58.152975941Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:58.163709707Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61054"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:23:58.211198889Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"55bfa9ba-fb62-43ad-8880-e6d5bf35932e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61054","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":46115,"response_code":"200"} {"msg":"wait for service Some(\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\") failed: Property not found. retry in 533.381878ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:58.266573563Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\") failed: Property not found. retry in 822.233399ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:58.415599925Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\") failed: Property not found. retry in 1.408408018s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:58.43148331Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\") failed: Property not found. retry in 1.17912414s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:58.52258845Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\") failed: Property not found. retry in 765.160002ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:58.540277349Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\") failed: Property not found. retry in 617.251204ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:58.550142489Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\") failed: Property not found. retry in 1.492432828s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:58.64910641Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\") failed: Property not found. retry in 1.499521097s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:58.78666158Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\") failed: Property not found. retry in 862.327999ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:58.796307476Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\") failed: Property not found. retry in 845.35604ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:58.804739568Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\") failed: Property not found. retry in 559.344205ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:58.812924055Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\") failed: Property not found. retry in 1.136739935s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:59.072595982Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\") failed: Property not found. retry in 1.324204655s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:59.145167376Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\") failed: Property not found. retry in 1.118286702s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:59.177264292Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\") failed: Property not found. retry in 1.119584492s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:59.221682485Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\") failed: Property not found. retry in 1.027621298s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:59.247064503Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\") failed: Property not found. retry in 581.453823ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:59.263756613Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\") failed: Property not found. retry in 540.296491ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:59.316507896Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\") failed: Property not found. retry in 1.039668401s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:59.382177558Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\") failed: Property not found. retry in 1.236502635s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:59.417955905Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\") failed: Property not found. retry in 840.052966ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:59.449100523Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\") failed: Property not found. retry in 758.55114ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:59.478952966Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\") failed: Property not found. retry in 1.004801292s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:59.568027177Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\") failed: Property not found. retry in 1.024292917s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:59.660952312Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\") failed: Property not found. retry in 897.493694ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:59.670369283Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\") failed: Property not found. retry in 725.160985ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:59.710985282Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\") failed: Property not found. retry in 762.265116ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:59.850183505Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\") failed: Property not found. retry in 723.08208ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:59.859365621Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\") failed: Property not found. retry in 629.619029ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:23:59.868718451Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\") failed: Property not found. retry in 610.889199ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:00.153872148Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\") failed: Property not found. retry in 636.114329ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:00.224113747Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\") failed: Property not found. retry in 1.216554909s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:00.252639535Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\") failed: Property not found. retry in 1.402316915s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:00.287410982Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\") failed: Property not found. retry in 1.168977823s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:00.298360003Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\") failed: Property not found. retry in 739.231581ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:00.308361455Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\") failed: Property not found. retry in 995.011769ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:00.318222754Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\") failed: Property not found. retry in 1.204915838s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:00.352306579Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\") failed: Property not found. retry in 663.294844ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:00.436038357Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\") failed: Property not found. retry in 999.904107ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:00.4508065Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\") failed: Property not found. retry in 673.262773ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:00.482785905Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\") failed: Property not found. retry in 1.41262271s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:00.51168923Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\") failed: Property not found. retry in 965.212809ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:00.580695575Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\") failed: Property not found. retry in 634.714023ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:00.593355168Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\") failed: Property not found. retry in 1.17586558s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:00.606101063Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\") failed: Property not found. retry in 976.134963ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:00.625914844Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\") failed: Property not found. retry in 1.48481011s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:00.667744057Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\") failed: Property not found. retry in 785.805427ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:00.714453613Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\") failed: Property not found. retry in 1.303494653s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:00.787923454Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\") failed: Property not found. retry in 1.486106169s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:00.877603547Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\") failed: Property not found. retry in 1.237522315s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:01.062384245Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\") failed: Property not found. retry in 1.435973693s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:01.111461288Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\") failed: Property not found. retry in 777.116387ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:01.167672408Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\") failed: Property not found. retry in 912.022755ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:01.241058767Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\") failed: Property not found. retry in 553.581293ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:01.326345075Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\") failed: Property not found. retry in 1.489769494s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:01.464382086Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\") failed: Property not found. retry in 973.715111ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:01.480545447Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\") failed: Property not found. retry in 952.019303ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:01.493881073Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\") failed: Property not found. retry in 1.107792988s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:01.515772683Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\") failed: Property not found. retry in 1.443412668s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:01.559053994Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\") failed: Property not found. retry in 778.352604ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:01.570393342Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\") failed: Property not found. retry in 1.187915093s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:01.615844025Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\") failed: Property not found. retry in 1.420070848s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:01.702748923Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\") failed: Property not found. retry in 1.201083657s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:01.796920782Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\") failed: Property not found. retry in 759.097101ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:01.893498237Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\") failed: Property not found. retry in 941.459788ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:01.937784557Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\") failed: Property not found. retry in 886.992287ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:01.957809452Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\") failed: Property not found. retry in 1.120834249s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:02.10558011Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\") failed: Property not found. retry in 594.743653ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:02.165506391Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\") failed: Property not found. retry in 1.358634584s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:02.177191965Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\") failed: Property not found. retry in 1.182923475s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:02.312475553Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\") failed: Property not found. retry in 895.052434ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:02.361420953Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\") failed: Property not found. retry in 616.461255ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:02.376188567Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\") failed: Property not found. retry in 1.153304824s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:02.457454878Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\") failed: Property not found. retry in 615.798235ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:02.467723395Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\") failed: Property not found. retry in 1.212016585s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:02.559800303Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\") failed: Property not found. retry in 656.609516ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:02.638469564Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\") failed: Property not found. retry in 782.104812ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:02.666257228Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\") failed: Property not found. retry in 793.750924ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:02.777908282Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\") failed: Property not found. retry in 1.264195956s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:02.819516461Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\") failed: Property not found. retry in 826.126549ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:02.856915769Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\") failed: Property not found. retry in 1.381291575s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:02.892323039Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\") failed: Property not found. retry in 1.148446458s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:02.96686252Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\") failed: Property not found. retry in 785.58963ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:03.006295278Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\") failed: Property not found. retry in 991.660283ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:03.016906842Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\") failed: Property not found. retry in 1.08911745s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:03.026788611Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\") failed: Property not found. retry in 1.042974642s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:03.09442981Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\") failed: Property not found. retry in 1.172046739s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:03.133134094Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\") failed: Property not found. retry in 523.777331ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:03.236891707Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\") failed: Property not found. retry in 1.300391669s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:03.2672797Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\") failed: Property not found. retry in 838.461032ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:03.304541016Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\") failed: Property not found. retry in 936.776655ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:03.474690414Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\") failed: Property not found. retry in 1.228278986s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:03.525399448Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\") failed: Property not found. retry in 1.176189887s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:03.588560721Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\") failed: Property not found. retry in 739.723551ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:03.598892129Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\") failed: Property not found. retry in 799.741726ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:03.620974653Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\") failed: Property not found. retry in 1.126818281s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:03.692220791Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\") failed: Property not found. retry in 699.608463ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:03.769516855Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\") failed: Property not found. retry in 1.110189936s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:03.780481236Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\") failed: Property not found. retry in 972.548068ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:03.801546821Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\") failed: Property not found. retry in 1.433206645s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:04.020034667Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\") failed: Property not found. retry in 500.22847ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:04.095766671Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\") failed: Property not found. retry in 536.336056ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:04.124310819Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\") failed: Property not found. retry in 1.169096925s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:04.131556439Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\") failed: Property not found. retry in 1.197834233s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:04.147637427Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\") failed: Property not found. retry in 1.126353041s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:04.156577859Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\") failed: Property not found. retry in 1.108431009s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:04.282994617Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\") failed: Property not found. retry in 1.283513581s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:04.316612702Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\") failed: Property not found. retry in 1.337269311s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:04.348725009Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\") failed: Property not found. retry in 1.265816659s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:04.420487437Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\") failed: Property not found. retry in 844.59216ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:04.429289637Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\") failed: Property not found. retry in 1.188004207s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:04.477119655Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\") failed: Property not found. retry in 1.230243843s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:04.576691628Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\") failed: Property not found. retry in 883.6763ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:04.604744966Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\") failed: Property not found. retry in 1.009015694s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:04.669929048Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\") failed: Property not found. retry in 1.127060191s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:04.764520004Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\") failed: Property not found. retry in 901.842221ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:04.774453735Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\") failed: Property not found. retry in 925.855269ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:04.783483118Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\") failed: Property not found. retry in 1.499385179s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:04.828973692Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\") failed: Property not found. retry in 842.856631ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:04.90334111Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\") failed: Property not found. retry in 1.187488229s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:05.289331053Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\") failed: Property not found. retry in 1.490762334s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:05.301352604Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\") failed: Property not found. retry in 792.074109ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:05.315783311Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\") failed: Property not found. retry in 1.085773673s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:05.357759078Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\") failed: Property not found. retry in 934.376468ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:05.405870542Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\") failed: Property not found. retry in 1.358680938s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:05.468065546Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\") failed: Property not found. retry in 716.742334ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:05.506064616Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Profile for oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:05.549387268Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:05.549861297Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\") failed: Property not found. retry in 1.140763594s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:05.620968023Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb\") failed: Property not found. retry in 1.445460042s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:05.685223667Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895\") failed: Property not found. retry in 783.491743ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:05.701916797Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\") failed: Property not found. retry in 1.447267844s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:05.721141816Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\") failed: Property not found. retry in 962.416144ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:05.735893679Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\") failed: Property not found. retry in 583.02277ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:05.756625017Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\") failed: Property not found. retry in 748.803949ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:05.785222397Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75\") failed: Property not found. retry in 1.025838415s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:05.805585508Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\") failed: Property not found. retry in 1.360517678s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:05.82496237Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\") failed: Property not found. retry in 44.943574ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:05.836819868Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\") failed: Property not found. retry in 78.38267ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:05.894514656Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\") failed: Property not found. retry in 608.250058ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:05.906916654Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\") failed: Property not found. retry in 203.947255ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:06.001313537Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\") failed: Property not found. retry in 1.237159367s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:06.122091996Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\") failed: Property not found. retry in 378.187938ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:06.220071638Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\") failed: Property not found. retry in 1.498926381s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:06.237820389Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\") failed: Property not found. retry in 511.018969ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:06.341110903Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\") failed: Property not found. retry in 560.303662ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:06.353749096Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\") failed: Property not found. retry in 889.253739ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:06.364950121Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\") failed: Property not found. retry in 1.007645036s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:06.456225444Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\") failed: Property not found. retry in 642.068062ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:06.494416337Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\") failed: Property not found. retry in 687.663125ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:06.531975768Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\") failed: Property not found. retry in 862.606725ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:06.56017177Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\") failed: Property not found. retry in 966.528622ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:06.615398691Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\") failed: Property not found. retry in 1.263223103s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:06.70963119Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\") failed: Property not found. retry in 1.022528761s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:06.776901351Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\") failed: Property not found. retry in 1.03777376s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:06.806627862Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\") failed: Property not found. retry in 963.458416ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:06.840013074Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\") failed: Property not found. retry in 1.173267713s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:06.878572434Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\") failed: Property not found. retry in 930.345671ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:06.931487551Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\") failed: Property not found. retry in 1.199533584s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:07.159550051Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\") failed: Property not found. retry in 1.064924393s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:07.180684737Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\") failed: Property not found. retry in 1.308371516s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:07.197851456Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\") failed: Property not found. retry in 1.207354578s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:07.234460399Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9\") failed: Property not found. retry in 1.41332603s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:07.266029796Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\") failed: Property not found. retry in 887.672203ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:07.371507641Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\") failed: Property not found. retry in 1.455549304s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:07.434864858Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663\") failed: Property not found. retry in 746.181486ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:07.477467826Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\") failed: Property not found. retry in 1.170867204s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:07.595181457Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885\") failed: Property not found. retry in 1.298928046s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:07.750446048Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca\") failed: Property not found. retry in 1.228040714s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:07.812029361Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\") failed: Property not found. retry in 995.068068ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:07.822323319Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\") failed: Property not found. retry in 1.195466658s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:07.856069737Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\") failed: Property not found. retry in 898.846288ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:07.873773757Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63\") failed: Property not found. retry in 981.115827ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:07.984433792Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\") failed: Property not found. retry in 506.197651ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:08.063350548Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:08.137379739Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"8.672001354s"} {"msg":"wait for service Some(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\") failed: Property not found. retry in 713.275415ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:08.25611396Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\") failed: Property not found. retry in 860.123238ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:08.268946056Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6\") failed: Property not found. retry in 1.447380218s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:08.370370134Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\") failed: Property not found. retry in 1.45706733s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:08.454450699Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\") failed: Property not found. retry in 1.016382394s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:08.516113193Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\") failed: Property not found. retry in 1.42559111s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:08.582300324Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\") failed: Property not found. retry in 770.660555ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:08.778747897Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0\") failed: Property not found. retry in 1.128520225s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:08.789449422Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7\") failed: Property not found. retry in 728.5689ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:08.830409159Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\") failed: Property not found. retry in 841.057854ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:08.905084133Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\") failed: Property not found. retry in 959.583053ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:08.996872496Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\") failed: Property not found. retry in 1.268635873s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:09.091747848Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\") failed: Property not found. retry in 624.450979ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:09.147208343Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\") failed: Property not found. retry in 850.526483ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:09.547935309Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\") failed: Property not found. retry in 1.014438471s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:09.562634542Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\") failed: Property not found. retry in 1.142136787s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:09.761862777Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\") failed: Property not found. retry in 803.502087ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:09.790356004Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\") failed: Property not found. retry in 582.425718ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:09.927261434Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\") failed: Property not found. retry in 655.900779ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:09.969193009Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\") failed: Property not found. retry in 957.190942ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:10.02336825Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\") failed: Property not found. retry in 1.199620523s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:10.374541604Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\") failed: Property not found. retry in 820.40878ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:10.416802935Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\") failed: Property not found. retry in 1.256371281s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:10.525939382Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\") failed: Property not found. retry in 1.325725702s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:10.589740287Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\") failed: Property not found. retry in 1.126904669s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:10.607246793Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\") failed: Property not found. retry in 1.194081173s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:10.639962111Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\") failed: Property not found. retry in 589.634209ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:10.916809707Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e\") failed: Property not found. retry in 1.387132871s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:10.993323247Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\") failed: Property not found. retry in 1.08902539s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:11.249920224Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a\") failed: Property not found. retry in 1.381520512s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:11.517285219Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\") failed: Property not found. retry in 573.560363ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:11.585268564Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b\") failed: Property not found. retry in 1.278151903s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:11.745378334Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d\") failed: Property not found. retry in 634.643236ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:11.794920529Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad\") failed: Property not found. retry in 1.029747979s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:11.845087802Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\") failed: Property not found. retry in 596.275083ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:11.927886643Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f\") failed: Property not found. retry in 623.247942ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:12.16940164Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\") failed: Property not found. retry in 1.241759009s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:12.352367564Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\") failed: Property not found. retry in 613.043074ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:12.536370498Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\") failed: Property not found. retry in 1.415751523s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:13.161571224Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f\") failed: Property not found. retry in 690.307473ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:14.589591058Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.308462673Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_nexus_1b5ada24-1ce5-4a0b-9c11-3c803e3e9270"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.308523804Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_nexus_1d7f4b38-ce97-4d7c-b37c-92212a02a45f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.308531034Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_pantry_58c7648a-2f27-407e-a7ac-2bf32aa3eb75"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.308541774Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_pantry_d41ab55b-e1a6-4706-8fb9-30f250e379ca"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.308546285Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_pantry_861d2408-b2e2-416d-a5c8-444212cd8663"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.308550494Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_nexus_bda5150c-3889-413e-a774-9d40d8f8536d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.308554415Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_02ac6d47-6979-4b37-b0b4-656336a6af63"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.308558425Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_325a9c1d-813c-4693-8cf2-b4f93c573895"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.308562535Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_bb490289-6569-4f8c-868b-d3dd7bd03ac6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.308566955Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_eb6ad2c3-d718-4b25-a9c1-d6170a395fb0"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.308573395Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_a243b058-fd7d-4081-979b-227b010730e9"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.308577255Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_77258649-3670-42da-8357-0d10bcdb4fa7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.308581285Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_clickhouse_227c6e26-7635-44f8-ab05-835d9cccc56f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.313713194Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_10fd3267-f358-4a92-b50e-b95e29dff885"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.313724304Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_75b8e0fb-4039-4278-a9cd-e3a5e1e7cecb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.313743774Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_43273bd0-1810-4edd-be2e-b5a92667be8a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.313748574Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_oximeter_0a365f9e-6c70-4a27-ba2b-6123ad53013e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.313871097Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_external_dns_f4635fcc-61fc-4ff9-8203-b560522a972b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.313877907Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_external_dns_dc5f3810-93b6-44f1-815c-44411abf99ad"} {"msg":"Writing ledger to /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.314518639Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.337152904Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.363460609Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"08c89d92-6d3d-46f8-887c-5885aa3daee8","remote_addr":"[fd00:1122:3344:101::1]:39593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":38924651,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.406547497Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1415"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.409554874Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:790"} {"msg":"Nexus address: [fd00:1122:3344:101::b]:12221","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.428265784Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:803"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n lldp: None,\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n rib_priority: None,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n tx_eq: None,\n uplink_port_fec: Some(\n None,\n ),\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.468622449Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1017"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/79114303-f678-43f7-aff3-a882c8c99560/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.509133717Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/79114303-f678-43f7-aff3-a882c8c99560/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.566634911Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/79114303-f678-43f7-aff3-a882c8c99560/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.733115677Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/79114303-f678-43f7-aff3-a882c8c99560/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:15.925843619Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/79114303-f678-43f7-aff3-a882c8c99560/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:16.122357363Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/79114303-f678-43f7-aff3-a882c8c99560/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:16.245927086Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/79114303-f678-43f7-aff3-a882c8c99560/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:16.597818164Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/79114303-f678-43f7-aff3-a882c8c99560/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:16.770184764Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:24:16.865735319Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"13.939301588s"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/79114303-f678-43f7-aff3-a882c8c99560/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:17.067979583Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/79114303-f678-43f7-aff3-a882c8c99560/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:17.611469631Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/79114303-f678-43f7-aff3-a882c8c99560/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:17.945844892Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/79114303-f678-43f7-aff3-a882c8c99560/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:18.816040523Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/79114303-f678-43f7-aff3-a882c8c99560/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:19.259840027Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3c806e15-5eb9-4848-92e3-601447cf12e5\", \"content-length\": \"173\", \"date\": \"Thu, 16 Jan 2025 23:24:23 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"79114303-f678-43f7-aff3-a882c8c99560\\\"\", request_id: \"3c806e15-5eb9-4848-92e3-601447cf12e5\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:23.340383902Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1b50a333-d2d0-4df8-89ab-7e5e692c05ec\", \"content-length\": \"173\", \"date\": \"Thu, 16 Jan 2025 23:24:23 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"79114303-f678-43f7-aff3-a882c8c99560\\\"\", request_id: \"1b50a333-d2d0-4df8-89ab-7e5e692c05ec\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:24.55302446Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ea6603c3-5f94-40f5-ae73-94e90d1351ea\", \"content-length\": \"173\", \"date\": \"Thu, 16 Jan 2025 23:24:25 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"79114303-f678-43f7-aff3-a882c8c99560\\\"\", request_id: \"ea6603c3-5f94-40f5-ae73-94e90d1351ea\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:25.70226341Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:28.259842438Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41339"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:28.25991045Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"47b83e5e-ece3-4534-92ae-c76c7ad708c9","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41339","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":28,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"10d04b58-1134-4176-92ff-8b3d04162361\", \"content-length\": \"173\", \"date\": \"Thu, 16 Jan 2025 23:24:27 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"198593b5-c3c6-4ef5-879a-52095a2ef511\\\"\", request_id: \"10d04b58-1134-4176-92ff-8b3d04162361\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:28.351955778Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8cbe20ae-1acc-47ca-a7d4-1067e0780f50\", \"content-length\": \"173\", \"date\": \"Thu, 16 Jan 2025 23:24:30 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"198593b5-c3c6-4ef5-879a-52095a2ef511\\\"\", request_id: \"8cbe20ae-1acc-47ca-a7d4-1067e0780f50\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:31.146868843Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b70618e4-7d76-49fd-a0f3-0a60ce760675\", \"content-length\": \"173\", \"date\": \"Thu, 16 Jan 2025 23:24:32 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"198593b5-c3c6-4ef5-879a-52095a2ef511\\\"\", request_id: \"b70618e4-7d76-49fd-a0f3-0a60ce760675\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:32.813052301Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2bd1055a-6199-403d-9285-bbfb62f5cd67\", \"content-length\": \"173\", \"date\": \"Thu, 16 Jan 2025 23:24:35 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"198593b5-c3c6-4ef5-879a-52095a2ef511\\\"\", request_id: \"2bd1055a-6199-403d-9285-bbfb62f5cd67\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:35.153866706Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"69e44fde-e65c-4f6e-8720-46f2adf56d43\", \"content-length\": \"173\", \"date\": \"Thu, 16 Jan 2025 23:24:37 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"198593b5-c3c6-4ef5-879a-52095a2ef511\\\"\", request_id: \"69e44fde-e65c-4f6e-8720-46f2adf56d43\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:37.513474912Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c0f4b993-8a58-4880-8866-cfd8b263f9b4\", \"content-length\": \"173\", \"date\": \"Thu, 16 Jan 2025 23:24:40 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"198593b5-c3c6-4ef5-879a-52095a2ef511\\\"\", request_id: \"c0f4b993-8a58-4880-8866-cfd8b263f9b4\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:40.191666176Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fff1be36-1936-4c4c-8d53-28d30a31ae3a\", \"content-length\": \"173\", \"date\": \"Thu, 16 Jan 2025 23:24:42 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"198593b5-c3c6-4ef5-879a-52095a2ef511\\\"\", request_id: \"fff1be36-1936-4c4c-8d53-28d30a31ae3a\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:43.00316111Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8f260ce7-427f-4afd-9e8b-9bb5f6c25898\", \"content-length\": \"173\", \"date\": \"Thu, 16 Jan 2025 23:24:48 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"198593b5-c3c6-4ef5-879a-52095a2ef511\\\"\", request_id: \"8f260ce7-427f-4afd-9e8b-9bb5f6c25898\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:49.247909319Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d35bfe37-aa74-41ad-8f22-c6fad6a9d45b\", \"content-length\": \"173\", \"date\": \"Thu, 16 Jan 2025 23:24:53 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"198593b5-c3c6-4ef5-879a-52095a2ef511\\\"\", request_id: \"d35bfe37-aa74-41ad-8f22-c6fad6a9d45b\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:53.454368253Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:58.318897276Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35639"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:58.319085819Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"73e1cc71-df2e-4440-bb7d-b1119f8316e2","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35639","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":25,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8d3d71bc-2b6c-4d78-8c5a-efe149312f89\", \"content-length\": \"173\", \"date\": \"Thu, 16 Jan 2025 23:24:58 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"198593b5-c3c6-4ef5-879a-52095a2ef511\\\"\", request_id: \"8d3d71bc-2b6c-4d78-8c5a-efe149312f89\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:24:58.660251122Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:05.723184893Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::d]:59260"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:05.768915842Z","hostname":"buskin","pid":1566,"uri":"/198593b5-c3c6-4ef5-879a-52095a2ef511","method":"GET","req_id":"e38d3dbc-24be-4c52-8bec-19272f64da88","remote_addr":"[fd00:1122:3344:101::d]:59260","local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":45295,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"575d69de-08db-4540-8d2c-c8fa654bbb8e\", \"content-length\": \"173\", \"date\": \"Thu, 16 Jan 2025 23:25:06 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"198593b5-c3c6-4ef5-879a-52095a2ef511\\\"\", request_id: \"575d69de-08db-4540-8d2c-c8fa654bbb8e\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:06.797437684Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1079"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:11.831624244Z","hostname":"buskin","pid":1566,"component":"NexusNotifierTask","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/nexus.rs:500","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:11.918217368Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:62425"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:11.920593373Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:11.921169184Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:11.92149225Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:11.922404468Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:11.936250374Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:11.936403457Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:11.957170656Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:11.957236467Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:11.978507816Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:11.978579387Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_736d5541-d5bd-460c-a5f5-441d6996aa03"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:12.000282774Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:12.000330615Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_65ae8c88-c7ba-40ff-af14-085d448fc3c0"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:12.022956789Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:12.02301936Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_9764f33b-5a77-475e-83b5-777615774b6a"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:12.044956822Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:12.045018593Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_28fb7d98-1494-4570-b25a-aa56857fbe19"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:12.065713651Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:12.065772772Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_723d2041-12e1-48e9-aa34-7148357fe80c"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:12.087697693Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:12.087757854Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_7980b762-6ad3-435b-bf8d-91d18dcf75d4"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:12.107760828Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:12.107811759Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:12.120611875Z","hostname":"buskin","pid":1566,"uri":"/inventory","method":"GET","req_id":"cc1f3760-bb48-41ae-8190-10f45394f669","remote_addr":"[fd00:1122:3344:101::c]:62425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":202188,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:21.958285655Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:42416"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:21.963825072Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:21.964001365Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:21.964168018Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:21.964189059Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:21.96422823Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:21.96424829Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:21.96425981Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:21.964288671Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"09607264-0e5d-452f-a8e2-f27b1088de69","remote_addr":"[fd00:1122:3344:101::b]:42416","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5907,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:21.965986553Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1089"} {"msg":"Writing ledger to /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:21.969674924Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:21.9699557Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.155524984Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:37906"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.155668626Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.155679636Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.155713627Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.155991362Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.168682626Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.168741657Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.192483943Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.192542664Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.202305462Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:64000"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.202633258Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.202688579Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.20272409Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.20273837Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.202759301Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.202781611Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.202800401Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.202827612Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"71f2501d-15b6-47d0-b0be-70a787e34701","remote_addr":"[fd00:1122:3344:101::a]:64000","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":398,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.218713967Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.218773838Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_736d5541-d5bd-460c-a5f5-441d6996aa03"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.238996227Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:46708"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.239314483Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.239374604Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.239449745Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.239469296Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.239479686Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.239499256Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.239506646Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.239525587Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"ee325cc9-a530-45f6-a9d3-38935b3049c8","remote_addr":"[fd00:1122:3344:101::c]:46708","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":412,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.243639626Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.243664226Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_65ae8c88-c7ba-40ff-af14-085d448fc3c0"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.265503066Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.265562907Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_9764f33b-5a77-475e-83b5-777615774b6a"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.28604633Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.286096401Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_28fb7d98-1494-4570-b25a-aa56857fbe19"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.304903252Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.304956573Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_723d2041-12e1-48e9-aa34-7148357fe80c"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.326280963Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.326338594Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_7980b762-6ad3-435b-bf8d-91d18dcf75d4"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.346146245Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.346200675Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.358445701Z","hostname":"buskin","pid":1566,"uri":"/inventory","method":"GET","req_id":"f12a68c4-0e46-41e3-93ea-b5841e3121ec","remote_addr":"[fd00:1122:3344:101::b]:37906","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":202807,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.620417022Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:62061"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.620736068Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.62080574Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.62083276Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.620857281Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.620888741Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.620898131Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.620915631Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:22.620944532Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"045b1804-a78c-470b-9d9a-85b14ef73342","remote_addr":"[fd00:1122:3344:101::b]:62061","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":409,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.319757812Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:49863"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.320138599Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"5d214482-6546-472d-af06-46a2a3745abb","remote_addr":"[fd00:1122:3344:101::b]:49863","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":323,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.369177181Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:48881"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.369279253Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"b0a0bdad-ab24-4af7-9454-01f62098c91d","remote_addr":"[fd00:1122:3344:101::c]:48881","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":36,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.570550368Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.570586019Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"08b2c7ca-1241-42b4-92b0-2b63b0447c57","remote_addr":"[fd00:1122:3344:101::c]:48881","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":407,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.58575935Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:49097"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.58780567Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.683074689Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.758678961Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"29d737f6-587c-496f-ab47-3e2f1e2af9bd","remote_addr":"[fd00:1122:3344:101::c]:49097","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":172825,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.758879824Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:52619"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.758963986Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:45684"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.759331123Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.759345244Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.759403905Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"c81c2023-c980-488f-b8e9-f39ebc9942ab","remote_addr":"[fd00:1122:3344:101::b]:49863","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":314,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.764245707Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"fa69b541-70b8-4bbe-9e3b-a3d1da283dee","remote_addr":"[fd00:1122:3344:101::b]:52619","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4906,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.764586664Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"364769d5-17df-406f-b28f-cbad6535c2b0","remote_addr":"[fd00:1122:3344:101::c]:49097","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5538,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.767296076Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:33163"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.767549741Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"e15dfc15-3ed1-4423-851d-932c9faab36d","remote_addr":"[fd00:1122:3344:101::c]:33163","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":185,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.823161439Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.851429152Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/77444133-e801-4aba-9330-b96b8b00119b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.851568855Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.852242228Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.853522232Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.853595783Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"PUT","req_id":"a2e7e6c3-0272-4397-bdce-ec81f38fe21f","remote_addr":"[fd00:1122:3344:101::c]:49097","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3201,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.891563603Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"44a0341a-ce07-4766-ae46-008b12e9cac0","remote_addr":"[fd00:1122:3344:101::b]:49863","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":2205,"response_code":"204"} {"msg":"skipping delta compute for subnet","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.891738716Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:486","new_vers":"RouterVersion { router_id: 001de000-074c-4000-8000-000000000001, version: 4 }","old_vers":"RouterVersion { router_id: 001de000-074c-4000-8000-000000000001, version: 4 }","subnet":"RouterId { vni: Vni(100), kind: System }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.891779497Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"4575b60b-d4bc-4f62-bc76-57690165d543","remote_addr":"[fd00:1122:3344:101::c]:48881","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":158,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.907275934Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"7d504262-f8ec-4d69-bbfb-ff79141066ca","remote_addr":"[fd00:1122:3344:101::b]:45684","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":148083,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:27.907654402Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"ffcae487-e8a7-4089-9d05-b0bed61f043d","remote_addr":"[fd00:1122:3344:101::b]:45684","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":159,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:28.356326289Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48742"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:28.35641878Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"71be4e28-0480-4640-becf-1a934853ae11","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48742","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":29,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:29.798976464Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:43206"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:29.799103066Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"b4e41720-b5fc-417d-a8c9-cd11efe7015c","remote_addr":"[fd00:1122:3344:101::b]:43206","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":48,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:29.854434889Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:29.85448036Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"2b8e80d1-5486-41f8-8b09-191e225ecf88","remote_addr":"[fd00:1122:3344:101::b]:43206","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:29.863276019Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"43fd3a60-569c-4df9-81e1-1d72b4050b73","remote_addr":"[fd00:1122:3344:101::b]:43206","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":179,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:30.797023781Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:39431"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:30.797114533Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"9d29cf70-d68e-4fc5-a7c7-4f77a73c9666","remote_addr":"[fd00:1122:3344:101::a]:39431","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":36,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:30.863152421Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:30.863209482Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"0775d7ef-1d78-4648-b181-4ca3957a17e8","remote_addr":"[fd00:1122:3344:101::a]:39431","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":130,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:31.087898198Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:56999"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:31.088111412Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"35b8822f-d78b-4d25-bb41-f341ea469be3","remote_addr":"[fd00:1122:3344:101::a]:56999","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":136,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:31.095871531Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"db304716-8772-4ded-bf5d-344309c1eea9","remote_addr":"[fd00:1122:3344:101::a]:39431","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":174,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:31.180015327Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:51876"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:31.180120679Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:31.219624607Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:31.283083206Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"629855be-df98-42e3-8958-50ad6ba2df8d","remote_addr":"[fd00:1122:3344:101::a]:51876","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":102991,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:31.283500254Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"6e2f3a06-431f-4217-bd50-b45b6b1e5632","remote_addr":"[fd00:1122:3344:101::a]:51876","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":108,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:35.731695051Z","hostname":"buskin","pid":1566,"uri":"/198593b5-c3c6-4ef5-879a-52095a2ef511","method":"GET","req_id":"ec8481d0-2195-4175-82c8-e1d17d4c99da","remote_addr":"[fd00:1122:3344:101::d]:59260","local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7766,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:57.022945255Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:49227"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:57.023104738Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"554faf84-b4e5-4e5f-8ac0-a112a6b031be","remote_addr":"[fd00:1122:3344:101::b]:49227","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":65,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:57.078099254Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:57.078136205Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"05412706-5ede-48d2-a27c-1b987c86b7c7","remote_addr":"[fd00:1122:3344:101::b]:49227","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":72,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:57.089239788Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"3f96cfc2-98e9-4743-b48f-b911edbef2ec","remote_addr":"[fd00:1122:3344:101::b]:49227","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":183,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:57.133272314Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:63060"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:57.133374756Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"317f6257-6f5d-4b02-a8bb-679e65e8dd10","remote_addr":"[fd00:1122:3344:101::c]:63060","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":63,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:57.168039841Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:40974"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:57.168191434Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:57.210633069Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:57.264791539Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"c00b29d0-e210-42c4-afdf-ee5ef3c5ab32","remote_addr":"[fd00:1122:3344:101::b]:40974","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96646,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:57.265066025Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:65342"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:57.265138126Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:36356"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:57.265439222Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:57.265471782Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"65e546c1-9778-4e2e-85e4-27322e10f879","remote_addr":"[fd00:1122:3344:101::b]:40974","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":309,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:57.265482933Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"1e368eba-0164-4278-b4ea-6f43b9bc9488","remote_addr":"[fd00:1122:3344:101::c]:63060","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":303,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:57.265582444Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"0cd28697-779a-468a-af87-c61369c51c73","remote_addr":"[fd00:1122:3344:101::b]:65342","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":271,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:57.265596995Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"65597308-c64a-4247-ab2e-54654837c2e6","remote_addr":"[fd00:1122:3344:101::c]:36356","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":255,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:57.276739169Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"d114c1da-8b15-4da8-840e-438c32a06117","remote_addr":"[fd00:1122:3344:101::c]:63060","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":331,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:57.310405065Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:59217"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:57.310509097Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:57.349758231Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:57.406117954Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"c4ac8e1b-6f98-4139-968f-5a89eb60f0f9","remote_addr":"[fd00:1122:3344:101::c]:59217","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95641,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:57.406413089Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"35588399-5012-4cf5-9b56-7bee0a618a9a","remote_addr":"[fd00:1122:3344:101::c]:59217","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":111,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:58.392102239Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46103"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:25:58.392529627Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"5391dd80-bd5c-413e-bd6b-edc561bb25a3","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46103","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":29,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:00.572271148Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:57499"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:00.572424151Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"903dfea7-3399-4384-b7dc-a210d4016518","remote_addr":"[fd00:1122:3344:101::a]:57499","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:00.629854044Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:00.629876214Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"bd1296e8-067c-4f81-a66d-eb945b9fa4fc","remote_addr":"[fd00:1122:3344:101::a]:57499","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:00.696692108Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50255"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:00.696890952Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"d2660183-0dce-4b0b-98e8-4418499d5771","remote_addr":"[fd00:1122:3344:101::a]:50255","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":125,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:00.700642313Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"fc066565-c008-44ce-b89c-6e8129bee797","remote_addr":"[fd00:1122:3344:101::a]:57499","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":156,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:00.786517052Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:33989"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:00.786615374Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:00.826113893Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:00.881437255Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"f9993524-be1c-49fd-a8d5-6fb42543896b","remote_addr":"[fd00:1122:3344:101::a]:33989","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94861,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:00.881899664Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"0cc191c5-f040-4be9-9366-41400ecc7801","remote_addr":"[fd00:1122:3344:101::a]:33989","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":199,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:05.732534378Z","hostname":"buskin","pid":1566,"uri":"/198593b5-c3c6-4ef5-879a-52095a2ef511","method":"GET","req_id":"99f17b73-ed2f-4138-a93f-67606f169650","remote_addr":"[fd00:1122:3344:101::d]:59260","local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7761,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:27.18840288Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:60216"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:27.188597524Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"2d7f71b7-cb4e-4fe4-8900-648cb336fd63","remote_addr":"[fd00:1122:3344:101::b]:60216","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:27.245059188Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:27.245085369Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"816ca6f0-d558-4a98-a16b-4b58282eeca7","remote_addr":"[fd00:1122:3344:101::b]:60216","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":84,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:27.278891428Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:65035"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:27.279075921Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"5c36d32d-8f70-45d8-a6b3-c3c0ed7d2e17","remote_addr":"[fd00:1122:3344:101::b]:65035","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":120,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:27.282255522Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"52b3dee0-7717-4be5-a51e-db60cea6c9cb","remote_addr":"[fd00:1122:3344:101::b]:60216","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":164,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:27.361385872Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:51159"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:27.361494094Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"0247052b-2bd2-4b3d-bad9-f4e9f2ac6432","remote_addr":"[fd00:1122:3344:101::c]:51159","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":26,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:27.395316214Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:52538"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:27.395425286Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:27.433606999Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:27.491174215Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"9ef13f5d-feca-4de9-80ae-b0b85568fc7f","remote_addr":"[fd00:1122:3344:101::b]:52538","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95785,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:27.491366529Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:27.491379999Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"4d986e37-bd4c-4fcf-a496-59c8a52a05b7","remote_addr":"[fd00:1122:3344:101::c]:51159","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":71,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:27.491534572Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"777008b8-aee0-4f19-a76c-deb8ac4d9ee7","remote_addr":"[fd00:1122:3344:101::b]:52538","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:27.502347009Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"c699b3c6-3e8b-4142-98f4-30d9b055d46b","remote_addr":"[fd00:1122:3344:101::c]:51159","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":377,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:27.516152795Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:53332"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:27.516319488Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"0b591b18-82b5-42f4-80a7-4e3e9a52f502","remote_addr":"[fd00:1122:3344:101::c]:53332","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":124,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:27.654213756Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:51049"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:27.654319098Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:27.692133994Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:27.746368026Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"7f471953-a637-42a9-8a25-76dd2a78af8d","remote_addr":"[fd00:1122:3344:101::c]:51049","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":92093,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:27.746673501Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"a4d1c155-0500-4f1c-98b0-f6fba8d2d616","remote_addr":"[fd00:1122:3344:101::c]:51049","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":126,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:28.428237681Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38838"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:28.428846453Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"db0db2f1-3b3d-49ca-b2f5-b8ddee630cdf","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38838","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":35,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:30.653800403Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:56676"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:30.654045657Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"f42b73e2-3186-4464-a870-7cc1f2f78319","remote_addr":"[fd00:1122:3344:101::a]:56676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":174,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:30.730093487Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:38403"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:30.730182079Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"a386a7d7-c406-4bbe-bbff-9dd224ffa014","remote_addr":"[fd00:1122:3344:101::a]:38403","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":43,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:30.770627586Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:30.770650496Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"64055f1b-df04-4bc6-a7b3-616dddea1c03","remote_addr":"[fd00:1122:3344:101::a]:38403","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":49,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:30.780901553Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"0856fe1b-ca2d-40e7-987f-d4f1894391b8","remote_addr":"[fd00:1122:3344:101::a]:38403","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":146,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:31.002490769Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:59025"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:31.002590921Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:31.042772612Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:31.102899357Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"2635cfb3-82b0-4b54-985a-e18dfce6e2d6","remote_addr":"[fd00:1122:3344:101::a]:59025","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100353,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:31.103240154Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"c6adf32b-46b3-41ec-b493-611995a2db8b","remote_addr":"[fd00:1122:3344:101::a]:59025","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":120,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:35.733025297Z","hostname":"buskin","pid":1566,"uri":"/198593b5-c3c6-4ef5-879a-52095a2ef511","method":"GET","req_id":"8b029f93-8e88-4c8c-9876-a6222b408e4f","remote_addr":"[fd00:1122:3344:101::d]:59260","local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7951,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:44.658906747Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:57.000777198Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:61773"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:57.000951302Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"8e0a1091-2e74-4a88-89a5-b58eb341e81a","remote_addr":"[fd00:1122:3344:101::b]:61773","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":65,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:57.072666309Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:57.07269952Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"b2662c05-2cbc-414f-9c3b-ddc1e3317c2d","remote_addr":"[fd00:1122:3344:101::b]:61773","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":81,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:57.084314353Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"ac12742f-8e19-42e7-bae1-f8f60282e753","remote_addr":"[fd00:1122:3344:101::b]:61773","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":170,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:57.092427939Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:43159"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:57.092529Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:57.131588451Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:57.192892318Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"d99ad43a-0d32-4502-9d99-d0b0f6e5be52","remote_addr":"[fd00:1122:3344:101::b]:43159","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100411,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:57.193082652Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:51246"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:57.193118743Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:38662"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:57.193333287Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"7df25a9e-068b-486d-83f7-ffac7ba3c4c4","remote_addr":"[fd00:1122:3344:101::c]:38662","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":112,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:57.193442439Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"194665ce-510d-4544-b31d-9023909335f2","remote_addr":"[fd00:1122:3344:101::b]:43159","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":290,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:57.19352642Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"af171058-33e2-49f8-b93c-54db54f8622b","remote_addr":"[fd00:1122:3344:101::b]:51246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":217,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:57.236820592Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:57.236862832Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"534d5012-7089-4cf8-8b4a-4a616580ec92","remote_addr":"[fd00:1122:3344:101::c]:38662","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":105,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:57.246154661Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"498f5ba5-d603-4fea-a5ee-f959a3a2bd9a","remote_addr":"[fd00:1122:3344:101::c]:38662","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":279,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:57.313418233Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:48230"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:57.313589046Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"604f9ee2-36ae-463e-ac33-0eba11689ef4","remote_addr":"[fd00:1122:3344:101::c]:48230","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":114,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:57.315605625Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:63109"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:57.315685066Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:57.351565892Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:57.408833245Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"bb7bccea-bd15-4a5c-af6a-3a51cfbe2ce8","remote_addr":"[fd00:1122:3344:101::c]:63109","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93167,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:57.409097Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"2d0e8947-4f1c-4d05-823a-a25c9c3041e0","remote_addr":"[fd00:1122:3344:101::c]:63109","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:58.463569202Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60180"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:26:58.463662963Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"bcc92b01-f64f-4561-a48f-b819e8b3ab17","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60180","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":37,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:00.590421637Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:39974"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:00.59055101Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"c8360a54-a25e-4d4b-9318-8941c5211ec4","remote_addr":"[fd00:1122:3344:101::a]:39974","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":55,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:00.632454474Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:00.632473925Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"4ebe13f8-46a2-4aad-adef-49562d760de0","remote_addr":"[fd00:1122:3344:101::a]:39974","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":65,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:00.641285194Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"9a406e06-0408-4886-9af0-609e81dcf82e","remote_addr":"[fd00:1122:3344:101::a]:39974","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":183,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:00.705654779Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:57216"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:00.705800272Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"87b54abe-2ece-4344-ac42-923d6613748b","remote_addr":"[fd00:1122:3344:101::a]:57216","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:00.71558691Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:45152"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:00.715677372Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:00.751263065Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:00.802869476Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"082e17ad-2b34-4395-9d02-96fe0e609a4d","remote_addr":"[fd00:1122:3344:101::a]:45152","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":87225,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:00.803418377Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"61041995-f374-46fe-ba7c-cafc091a4fa4","remote_addr":"[fd00:1122:3344:101::a]:45152","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":206,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:05.733360135Z","hostname":"buskin","pid":1566,"uri":"/198593b5-c3c6-4ef5-879a-52095a2ef511","method":"GET","req_id":"003d1fe6-8572-47b0-a7e2-eed96a9b296e","remote_addr":"[fd00:1122:3344:101::d]:59260","local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6989,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:27.141245088Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:44152"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:27.141389451Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"c31f561e-ab5d-4864-9a51-2bd3a82b0628","remote_addr":"[fd00:1122:3344:101::b]:44152","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":55,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:27.186370535Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:27.186401436Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"1c3f6761-9813-49c3-9787-96da9e0c73aa","remote_addr":"[fd00:1122:3344:101::b]:44152","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":90,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:27.197258034Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"5743538e-772e-457f-ac8a-2097497989dd","remote_addr":"[fd00:1122:3344:101::b]:44152","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":233,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:27.225414835Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:53090"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:27.225532477Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"f4405c60-bc8d-46a7-a230-aa2e0601d6f6","remote_addr":"[fd00:1122:3344:101::c]:53090","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":47,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:27.301076148Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:27.301108539Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"65471da4-31a6-44d5-8ff8-791e8d023f94","remote_addr":"[fd00:1122:3344:101::c]:53090","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":54,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:27.312036008Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"aa251957-d84c-4003-b984-de22356f43b8","remote_addr":"[fd00:1122:3344:101::c]:53090","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":168,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:27.376222521Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:46145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:27.376420865Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"710f473c-e4ae-442a-be35-09bb9ab06097","remote_addr":"[fd00:1122:3344:101::b]:46145","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":145,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:27.491976544Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:53045"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:27.492199558Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"c4cfdb41-f80c-4df9-a039-9c4703d5283d","remote_addr":"[fd00:1122:3344:101::c]:53045","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":147,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:27.505492814Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:38624"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:27.505591556Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:27.542094377Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:27.59744299Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"5ae4449c-10ce-4ac1-99af-4cf395e55dc8","remote_addr":"[fd00:1122:3344:101::c]:38624","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91893,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:27.597583833Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41124"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:27.597758836Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:27.631062496Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:27.687100102Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"35836ee6-54b0-4548-acc9-6f68c71dc2f6","remote_addr":"[fd00:1122:3344:101::b]:41124","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89399,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:27.687368767Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"ca71259e-6bca-41c3-8fd9-0c3c3da526a3","remote_addr":"[fd00:1122:3344:101::c]:38624","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":121,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:27.687610171Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"91dd81ff-0ec3-4a0c-9386-3c39deaab5e6","remote_addr":"[fd00:1122:3344:101::b]:41124","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":154,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:28.496280261Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53840"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:28.496413484Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"898ddbe1-5d6e-4f06-9d60-40714cbd5528","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53840","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":63,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:30.623940873Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:64918"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:30.624148027Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"7a2ec21f-88f7-4cec-a72c-1be25a9325bc","remote_addr":"[fd00:1122:3344:101::a]:64918","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:30.684844002Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:30.684879243Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"4985c605-6d29-4dbe-b051-0f472df7e0c6","remote_addr":"[fd00:1122:3344:101::a]:64918","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":86,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:30.69824814Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"d67d0c01-2878-44ed-91f1-f65f31d63a55","remote_addr":"[fd00:1122:3344:101::a]:64918","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":228,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:30.871374354Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:53622"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:30.871532817Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"65ab9fa8-6e66-47de-becd-e8337b512285","remote_addr":"[fd00:1122:3344:101::a]:53622","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":120,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:30.880926088Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:49461"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:30.880996039Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:30.918343726Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:30.974959114Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"c7c464d3-36d7-450b-b486-5fba44baa6f0","remote_addr":"[fd00:1122:3344:101::a]:49461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93992,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:30.97526614Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"b4ca48b5-574b-49d0-aa23-11452670671b","remote_addr":"[fd00:1122:3344:101::a]:49461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:35.733857697Z","hostname":"buskin","pid":1566,"uri":"/198593b5-c3c6-4ef5-879a-52095a2ef511","method":"GET","req_id":"5542bab9-d6d7-4819-99c3-a506518237a1","remote_addr":"[fd00:1122:3344:101::d]:59260","local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7517,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:57.129033797Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:58988"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:57.129315472Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"5957f7a6-8660-4532-aaeb-2f61c27ef268","remote_addr":"[fd00:1122:3344:101::b]:58988","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":192,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:57.15107478Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:47289"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:57.151232923Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"303b15f4-5d0e-4902-9dda-66a708b6270c","remote_addr":"[fd00:1122:3344:101::c]:47289","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":65,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:57.167803882Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:59703"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:57.167944834Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"ee236dd3-0657-4a9c-8abf-1b5bd8fe5857","remote_addr":"[fd00:1122:3344:101::b]:59703","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":64,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:57.20415339Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:57.20417752Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"b57d2cf8-8f95-4c21-8697-6ad7a89eb7c0","remote_addr":"[fd00:1122:3344:101::c]:47289","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:57.213292595Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"35637cc6-7ed1-4212-9184-ebb3518f6d60","remote_addr":"[fd00:1122:3344:101::c]:47289","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":226,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:57.216442136Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:57.216464846Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"ca980647-9f5b-45db-bf02-6682a3f016d7","remote_addr":"[fd00:1122:3344:101::b]:59703","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":42,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:57.224659643Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"41dfea00-a146-4d64-8080-017782025a87","remote_addr":"[fd00:1122:3344:101::b]:59703","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":191,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:57.249775766Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:55592"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:57.249877798Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:57.294656658Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:57.354954686Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"9133179e-5f90-4ee0-a8aa-a6cf3a9803c4","remote_addr":"[fd00:1122:3344:101::b]:55592","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":105109,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:57.35518076Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:63418"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:57.355251391Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:37067"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:57.355527077Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"02a8750d-794b-4f40-aa16-c160379f91f2","remote_addr":"[fd00:1122:3344:101::b]:55592","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":289,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:57.355645369Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"06b335cc-f9b0-430b-8d1a-f65e31223a2d","remote_addr":"[fd00:1122:3344:101::c]:63418","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":251,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:57.35568903Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:57.394818722Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:57.450883028Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"14c9081f-5132-41b7-94fb-68dd46fd8a49","remote_addr":"[fd00:1122:3344:101::c]:37067","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95349,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:57.451276906Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"3d745792-0909-4e27-bced-3dbc0bdbe7da","remote_addr":"[fd00:1122:3344:101::c]:37067","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":151,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:58.529829159Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55185"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:27:58.529928421Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"c201aacc-89c8-474d-8f7b-aea85b7ab393","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55185","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":30,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:00.591568564Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:56604"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:00.591723267Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"25f828a1-0cab-4cae-8e75-97c23095f32a","remote_addr":"[fd00:1122:3344:101::a]:56604","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":87,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:00.639068056Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:00.639096117Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"9cd94ac5-1ca8-4b10-a0bd-39808e49c079","remote_addr":"[fd00:1122:3344:101::a]:56604","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":74,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:00.648135491Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"ab21885a-6eac-496f-950f-63b1d8a7dfce","remote_addr":"[fd00:1122:3344:101::a]:56604","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":171,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:00.775850333Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:51575"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:00.775990466Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"11a8ec25-21eb-4302-85f4-9c8737a2b71f","remote_addr":"[fd00:1122:3344:101::a]:51575","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:00.813331433Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:64112"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:00.813419575Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:00.853524715Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:00.917001504Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"8a863c16-054c-4f99-bc92-9955a1c52851","remote_addr":"[fd00:1122:3344:101::a]:64112","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103616,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:00.927265551Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"4f195615-abc5-4e2f-9e00-3c065c1c62ad","remote_addr":"[fd00:1122:3344:101::a]:64112","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:05.735358649Z","hostname":"buskin","pid":1566,"uri":"/198593b5-c3c6-4ef5-879a-52095a2ef511","method":"GET","req_id":"0ec30797-05d3-453a-b023-6ec893eabba2","remote_addr":"[fd00:1122:3344:101::d]:59260","local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9163,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:27.312339841Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:40233"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:27.312487704Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"579eaee7-0178-4cb1-8276-d63f959b2c5a","remote_addr":"[fd00:1122:3344:101::c]:40233","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":62,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:27.32995901Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:27.32997299Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"2193e818-3932-4197-8341-c659a6b55292","remote_addr":"[fd00:1122:3344:101::c]:40233","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":58,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:27.351558684Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:34821"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:27.351725898Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"93ae3bfc-e96f-4957-ae0c-a42028225251","remote_addr":"[fd00:1122:3344:101::b]:34821","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":113,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:27.359925415Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:32815"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:27.360053368Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"deb1834b-d0ad-40df-b821-d2a28f28a061","remote_addr":"[fd00:1122:3344:101::b]:32815","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":38,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:27.364378161Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:53094"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:27.364559904Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"82b7a325-bb98-47a6-9d45-b90e3ab9bb8e","remote_addr":"[fd00:1122:3344:101::c]:53094","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":132,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:27.373806492Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"d6c3a165-b076-40e4-a08d-d1ae91fb742e","remote_addr":"[fd00:1122:3344:101::c]:40233","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":200,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:27.404637614Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:27.404660054Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"e453ebfb-3b72-4b85-9913-c65baf42f474","remote_addr":"[fd00:1122:3344:101::b]:32815","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":63,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:27.413442553Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"e5d25f55-4b10-488d-b8b0-55377a7ac4ea","remote_addr":"[fd00:1122:3344:101::b]:32815","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":233,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:27.449523906Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:52205"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:27.449619458Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:27.48878887Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:27.543844497Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"6e0772a0-1e8b-4b8e-b228-ffbd80029730","remote_addr":"[fd00:1122:3344:101::b]:52205","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94268,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:27.544169874Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"c0a9de00-66d3-4f2c-a5f1-c75eba88ea48","remote_addr":"[fd00:1122:3344:101::b]:52205","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":106,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:27.558157352Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:43209"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:27.558507599Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:27.594873277Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:27.647690952Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"1a1d743c-d5a8-4e6c-bddf-15f93ce2ddcb","remote_addr":"[fd00:1122:3344:101::c]:43209","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89253,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:27.647952987Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"33e49262-daff-4f3d-b2b0-75b6ae3b8e66","remote_addr":"[fd00:1122:3344:101::c]:43209","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:28.564986198Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40567"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:28.56508301Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"72d2a986-6a48-4e02-8fff-d1087774d66f","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40567","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":32,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:30.723303878Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:41109"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:30.72344032Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"ec5fb687-8672-48d6-aa15-a5f085098b05","remote_addr":"[fd00:1122:3344:101::a]:41109","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":69,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:30.789175283Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:30.789243834Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"a19f4ff0-849b-402d-9737-9a2d3524341a","remote_addr":"[fd00:1122:3344:101::a]:41109","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":158,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:30.79786584Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"11268654-2342-4dc5-ac06-787ba6f54ef9","remote_addr":"[fd00:1122:3344:101::a]:41109","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":247,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:30.919254911Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:61723"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:30.919382954Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:30.959653977Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:31.02179329Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"ba3120dd-07f2-4a6e-8425-4fff48d9d347","remote_addr":"[fd00:1122:3344:101::a]:61723","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":102459,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:31.022107536Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:35484"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:31.022349041Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"2b2254e1-6b9c-41ae-ac08-864c049cfe77","remote_addr":"[fd00:1122:3344:101::a]:61723","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":233,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:31.022488354Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"985cd2cd-7717-4813-9229-747b43d2691f","remote_addr":"[fd00:1122:3344:101::a]:35484","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":229,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:35.733809404Z","hostname":"buskin","pid":1566,"uri":"/198593b5-c3c6-4ef5-879a-52095a2ef511","method":"GET","req_id":"40a7596a-8a13-4a6f-8e1a-48c6e46d9f51","remote_addr":"[fd00:1122:3344:101::d]:59260","local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7084,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:56.976953884Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:46189"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:56.977087007Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"d29723ee-3677-47d4-84a4-3daf7e150897","remote_addr":"[fd00:1122:3344:101::b]:46189","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":50,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:56.99285512Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:56.99287542Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"0b1e90ee-2a8d-4d8f-89b7-1ec848bc0961","remote_addr":"[fd00:1122:3344:101::b]:46189","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":57,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:57.003218469Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"90735686-4a73-470d-b411-70f7f62f429e","remote_addr":"[fd00:1122:3344:101::b]:46189","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":232,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:57.164879123Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:44716"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:57.165071837Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"3476e273-3b22-4a4d-9168-b611bc9e354d","remote_addr":"[fd00:1122:3344:101::b]:44716","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":145,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:57.186705213Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:59141"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:57.186798924Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:57.226399925Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:57.288267573Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"570a4cd9-5370-489c-8d10-d8df98a33c3e","remote_addr":"[fd00:1122:3344:101::b]:59141","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101512,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:57.288393306Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:34191"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:57.288510828Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"ac7f310f-0771-4a08-bdc7-93bc98887684","remote_addr":"[fd00:1122:3344:101::c]:34191","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":51,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:57.288656741Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"2dbf703a-a973-4ab6-99a1-b9cc4dffb568","remote_addr":"[fd00:1122:3344:101::b]:59141","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":92,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:57.33235063Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:38795"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:57.332621805Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"c494fdf5-e0aa-49b6-b10e-2fa1f5ccc5ef","remote_addr":"[fd00:1122:3344:101::c]:38795","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":194,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:57.333674245Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:57.333691486Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"76b6a5a7-5b3b-4fe4-a382-df671b7ac8a7","remote_addr":"[fd00:1122:3344:101::c]:34191","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":74,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:57.334855688Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:49116"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:57.33495202Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:57.370704076Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:57.42345282Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"cac02db8-eed4-4c4e-bb81-cf30a58d46d4","remote_addr":"[fd00:1122:3344:101::c]:49116","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88548,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:57.42762333Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"5417ef9f-5897-44d5-b10e-873bfba1e1e6","remote_addr":"[fd00:1122:3344:101::c]:49116","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3790,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:57.427743882Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"ab5135f3-d4b3-4569-bd07-f9b9822e09b2","remote_addr":"[fd00:1122:3344:101::c]:34191","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3838,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:58.598087999Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41279"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:28:58.598225531Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"374027d5-3c89-4518-a05f-b412292f4306","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41279","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":74,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:00.513937082Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:47745"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:00.514118655Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"6d32bf68-7f74-43a1-bf55-4b60b269a4f8","remote_addr":"[fd00:1122:3344:101::a]:47745","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:00.561057197Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:00.561126648Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"dd958c10-2826-4b3c-9051-df919308141f","remote_addr":"[fd00:1122:3344:101::a]:47745","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:00.575195898Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"b4622cf7-e348-4c33-ac76-83e35bf68536","remote_addr":"[fd00:1122:3344:101::a]:47745","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":280,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:00.615506983Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:34157"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:00.615608374Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:00.654034323Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:00.712199599Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"5e3e92c1-3339-4bbb-b94f-a5f7603d0833","remote_addr":"[fd00:1122:3344:101::a]:34157","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96638,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:00.712473824Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"a3d103e8-6883-445d-af59-654006da4df6","remote_addr":"[fd00:1122:3344:101::a]:34157","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:00.731385487Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:43158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:00.731601132Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"c02074f2-b3d0-48a3-8a59-09905f230b85","remote_addr":"[fd00:1122:3344:101::a]:43158","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":154,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:05.736083912Z","hostname":"buskin","pid":1566,"uri":"/198593b5-c3c6-4ef5-879a-52095a2ef511","method":"GET","req_id":"9f90c180-bc5d-4db1-9972-820122f0db11","remote_addr":"[fd00:1122:3344:101::d]:59260","local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8539,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.287896201Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:33837"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.288080445Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"d40a0dda-e65a-43f7-8c81-e91e3e9438ce","remote_addr":"[fd00:1122:3344:101::b]:33837","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":116,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.340215356Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.340252027Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"f6e8bbc4-dd69-4e6c-a53a-01a5779a5013","remote_addr":"[fd00:1122:3344:101::b]:33837","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":117,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.350132236Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"d92a39e4-ccd2-433f-9087-2d75e1f588ca","remote_addr":"[fd00:1122:3344:101::b]:33837","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":247,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.358450396Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:42865"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.358565918Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"391de229-353c-444d-8689-b76c55f9dffd","remote_addr":"[fd00:1122:3344:101::c]:42865","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":72,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.372570207Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:54760"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.37271339Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.419437097Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.483261363Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"dd02e64c-9116-4983-ab39-dd2a07a38999","remote_addr":"[fd00:1122:3344:101::b]:54760","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":110601,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.48362683Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.483650601Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"a80847e5-6e1d-41e2-adb6-f5156e87d643","remote_addr":"[fd00:1122:3344:101::c]:42865","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":141,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.483758503Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"e017e1d0-45d7-45e4-8043-eb341316ebf6","remote_addr":"[fd00:1122:3344:101::b]:54760","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":258,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.506292196Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:52349"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.506500109Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"1d3f9e03-459e-40c4-a736-f5b4d55a8d11","remote_addr":"[fd00:1122:3344:101::b]:52349","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":138,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.5455971Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:52613"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.545807894Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.545862255Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.545883866Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.545902006Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.545958707Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.545974638Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.545983908Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.545999858Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"849b018a-fc74-45df-81f2-611e1531e628","remote_addr":"[fd00:1122:3344:101::b]:52613","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":359,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.696310975Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:52511"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.696493398Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"41cbd883-74ec-4810-9822-2baca8d53d81","remote_addr":"[fd00:1122:3344:101::c]:52511","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.707915487Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"f095cdc6-dae5-42a3-85fa-a8d8d8af5574","remote_addr":"[fd00:1122:3344:101::c]:42865","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":252,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.731223855Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:34607"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.731332317Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.773761561Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.837944144Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"ec69420a-1a33-4ddd-8709-2dfacc457f6d","remote_addr":"[fd00:1122:3344:101::c]:34607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":106643,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:27.83823521Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"07fc1bdb-e99a-4999-b735-778adeabb171","remote_addr":"[fd00:1122:3344:101::c]:34607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":106,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:28.036827944Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:60702"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:28.03716415Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:28.037226621Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:28.037268072Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:28.037315403Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:28.037322563Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:28.037341224Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:28.037364764Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:28.037398815Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"d19e09a1-f2d3-4708-8365-3a9605789f62","remote_addr":"[fd00:1122:3344:101::c]:60702","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":453,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:28.631096997Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:28.63123811Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"b297a6ab-9f98-4b89-b8e9-90658afff822","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38196","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":46,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:30.733907581Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:43213"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:30.734105614Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"d287d05c-a935-4d84-aebe-8ae301ef55c0","remote_addr":"[fd00:1122:3344:101::a]:43213","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":118,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:30.785789427Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:30.785815747Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"56ba904b-0cbf-42be-a394-dad76a1bdf1c","remote_addr":"[fd00:1122:3344:101::a]:43213","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":87,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:30.795192308Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"f4c669ea-b85a-42cd-8a26-b1b24dad10b9","remote_addr":"[fd00:1122:3344:101::a]:43213","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":239,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:30.859208017Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:65419"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:30.859388451Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"4479b04c-cac3-4194-a2e6-efc2d8573bd8","remote_addr":"[fd00:1122:3344:101::a]:65419","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":129,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:30.874042552Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:38618"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:30.874178815Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:30.91612666Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:30.973988341Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"1b794203-63dc-4892-a89a-08b0120fa60c","remote_addr":"[fd00:1122:3344:101::a]:38618","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99853,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:30.974264557Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"d542887d-054c-4d97-bea1-e26a6cc2fd79","remote_addr":"[fd00:1122:3344:101::a]:38618","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":117,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:30.981621138Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:65526"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:30.981906864Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:30.981964655Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:30.982010016Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:30.982033426Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:30.982076607Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:30.982110177Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:30.982142978Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:30.982179099Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"572e3d13-01d7-4ac4-9ba6-0d9bf08f10f8","remote_addr":"[fd00:1122:3344:101::a]:65526","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":456,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:35.73417557Z","hostname":"buskin","pid":1566,"uri":"/198593b5-c3c6-4ef5-879a-52095a2ef511","method":"GET","req_id":"728835d7-ad52-4ded-a077-f3115e9af67a","remote_addr":"[fd00:1122:3344:101::d]:59260","local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7011,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:57.022775084Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:59522"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:57.022926037Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"c997df23-ff5a-44f1-b84d-0280148a42c2","remote_addr":"[fd00:1122:3344:101::b]:59522","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":61,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:57.099882865Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:57.099936836Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"8b77a5aa-8a80-438c-96da-3a01a73c2ad5","remote_addr":"[fd00:1122:3344:101::b]:59522","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":81,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:57.109861676Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"00b1c8bb-908c-47c7-a538-96ff38c5123b","remote_addr":"[fd00:1122:3344:101::b]:59522","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":234,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:57.180908471Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:40246"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:57.181073264Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"51615aa9-951b-4443-978e-e086a4763e61","remote_addr":"[fd00:1122:3344:101::b]:40246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":125,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:57.22249801Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:58069"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:57.222595881Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:57.262238393Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:57.318394712Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"fafe2da2-e2ad-4b0f-9274-852372ed1a12","remote_addr":"[fd00:1122:3344:101::b]:58069","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95837,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:57.318528734Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:47440"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:57.318709698Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"00cb0682-3e34-44a4-81e1-e2b4a06e1f29","remote_addr":"[fd00:1122:3344:101::c]:47440","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":87,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:57.318815729Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"d9c562b8-65eb-4395-a3c3-f1ff797c2d7e","remote_addr":"[fd00:1122:3344:101::b]:58069","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":143,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:57.363456437Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:57.363489807Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"9422cbc3-72d3-4d38-91b2-a98e19737d7b","remote_addr":"[fd00:1122:3344:101::c]:47440","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:57.388541989Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:54732"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:57.388751473Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"08a790ce-3ab6-406d-a7a9-071b37052a55","remote_addr":"[fd00:1122:3344:101::c]:54732","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":137,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:57.39121188Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"d8e7ec6d-6334-4294-a015-15603c786633","remote_addr":"[fd00:1122:3344:101::c]:47440","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":305,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:57.415761251Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:63674"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:57.415869464Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:57.451989907Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:57.504047917Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"af26f2e3-e345-4c00-90a3-c50003e186a3","remote_addr":"[fd00:1122:3344:101::c]:63674","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88211,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:57.506773229Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"1bf3bc7e-a2e8-4eb6-98d8-8db26b68e420","remote_addr":"[fd00:1122:3344:101::c]:63674","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:58.665335099Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65116"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:29:58.665446522Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"6f8f2d8c-a336-4e8d-982f-f35114a13745","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65116","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":39,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:00.585291232Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:49963"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:00.585454075Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"e1d1e683-cb64-4fea-8b66-50ab6b45c881","remote_addr":"[fd00:1122:3344:101::a]:49963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":76,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:00.640859729Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:00.64090511Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"a576eebb-b035-4835-98fb-36633ae58d2f","remote_addr":"[fd00:1122:3344:101::a]:49963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:00.651586365Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"6ed2ebf5-d094-4035-9d35-45d9615b6238","remote_addr":"[fd00:1122:3344:101::a]:49963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":250,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:00.677782168Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:42368"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:00.67790128Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:00.732319775Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:00.818178184Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"95be2448-7485-44f1-a9e6-e4e579396673","remote_addr":"[fd00:1122:3344:101::a]:42368","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":140319,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:00.81850736Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:33218"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:00.818813476Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"95ff5da2-3fd8-41ef-af4b-cae0fe480f8d","remote_addr":"[fd00:1122:3344:101::a]:42368","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":282,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:00.818914988Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"cab5a801-760e-42e5-9c67-65b7aaf7e0d3","remote_addr":"[fd00:1122:3344:101::a]:33218","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":229,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:05.735226625Z","hostname":"buskin","pid":1566,"uri":"/198593b5-c3c6-4ef5-879a-52095a2ef511","method":"GET","req_id":"06587fd2-b5c3-44ae-b6f8-b2f9be0515ea","remote_addr":"[fd00:1122:3344:101::d]:59260","local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6974,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:27.148178827Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:43097"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:27.14830951Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"1cf5f8de-1e3a-4243-ae51-e2290f844994","remote_addr":"[fd00:1122:3344:101::b]:43097","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":67,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:27.166356226Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:27.166400097Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"ee46b828-4c3d-43c6-9469-a3777ad5c8c9","remote_addr":"[fd00:1122:3344:101::b]:43097","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":69,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:27.174560884Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"e71edfba-2657-4fcd-befa-d1f842ca0a06","remote_addr":"[fd00:1122:3344:101::b]:43097","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":265,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:27.264373308Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:51743"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:27.264562062Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"bb83cfa0-ff77-4e35-867c-20e876c7d6d4","remote_addr":"[fd00:1122:3344:101::b]:51743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":141,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:27.336867441Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:52875"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:27.336968533Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:27.377221296Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:27.435351202Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"cc3c13fb-7930-4ed0-818d-1920055a3679","remote_addr":"[fd00:1122:3344:101::b]:52875","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98429,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:27.43577173Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"72328a36-9c42-40f5-ba96-97c5e61f0c50","remote_addr":"[fd00:1122:3344:101::b]:52875","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":157,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:27.540338239Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:52468"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:27.54042004Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"413e606a-3f14-44d8-b3d8-792c465cf667","remote_addr":"[fd00:1122:3344:101::c]:52468","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":47,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:27.554394778Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:27.554413669Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"3331dc0c-6c50-417f-8d4f-d49537c61a79","remote_addr":"[fd00:1122:3344:101::c]:52468","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":48,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:27.569858355Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:42046"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:27.570021359Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"76f8fcf4-8c94-4612-94a2-8298b7957670","remote_addr":"[fd00:1122:3344:101::c]:42046","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":113,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:27.579185805Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"6933d695-fa97-47d5-8fa8-7161a742103c","remote_addr":"[fd00:1122:3344:101::c]:52468","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":178,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:27.599417553Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:42402"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:27.599501685Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:27.63830877Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:27.692921669Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"32f5fb1f-0c06-4635-bbbc-57ec6dc7b626","remote_addr":"[fd00:1122:3344:101::c]:42402","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93448,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:27.693162534Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"7e1cf906-f54f-4927-8772-ce59ba358934","remote_addr":"[fd00:1122:3344:101::c]:42402","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:28.69895805Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60429"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:28.699103882Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"a499be03-c103-4f19-bb80-dfae12560c26","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60429","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":80,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:30.634300748Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:58611"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:30.634464731Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"f2df52aa-1fb6-4fcc-842f-6be954f23fd6","remote_addr":"[fd00:1122:3344:101::a]:58611","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:30.677388255Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:30.677403316Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"1be632eb-47b4-46cb-b536-2e4ded4077f7","remote_addr":"[fd00:1122:3344:101::a]:58611","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":75,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:30.686062212Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"999ca6ca-5e63-4fa6-9370-2edbb2cf5373","remote_addr":"[fd00:1122:3344:101::a]:58611","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":188,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:30.799998509Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:45634"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:30.800180653Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"8cbc5685-4a88-4e17-ade5-b4041266ebc1","remote_addr":"[fd00:1122:3344:101::a]:45634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":146,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:30.938282855Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:60169"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:30.938391597Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:30.980976335Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:31.040481538Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"7b6f204b-2734-470f-9e19-f5add810a6be","remote_addr":"[fd00:1122:3344:101::a]:60169","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":102141,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:31.040785874Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"f53dfc45-5c31-42a4-aeaa-ee422dc87c4c","remote_addr":"[fd00:1122:3344:101::a]:60169","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:35.736592596Z","hostname":"buskin","pid":1566,"uri":"/198593b5-c3c6-4ef5-879a-52095a2ef511","method":"GET","req_id":"2356ee22-5c8c-488f-ae6b-a9d40dcbeacd","remote_addr":"[fd00:1122:3344:101::d]:59260","local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8321,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:56.953117015Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:42084"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:56.953239438Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"6376f2bd-a9b4-43cc-aad1-71a7cc7bcfdf","remote_addr":"[fd00:1122:3344:101::b]:42084","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":54,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:57.005526652Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:57.005557063Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"2792be5f-4e1d-4296-9146-550c1c8b3fb0","remote_addr":"[fd00:1122:3344:101::b]:42084","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":83,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:57.013820201Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"e45cd8ac-7cda-435b-8c22-02035e8939fc","remote_addr":"[fd00:1122:3344:101::b]:42084","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":207,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:57.18087087Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:55384"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:57.181131475Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"df33b603-72b5-4c91-80ea-76cc1fbcc41d","remote_addr":"[fd00:1122:3344:101::b]:55384","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":198,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:57.185925117Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:45718"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:57.186011518Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"a474bca3-0bfc-4289-99b1-dc9b36d48aaf","remote_addr":"[fd00:1122:3344:101::c]:45718","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":43,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:57.207133314Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:57.207172625Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"d7c98bee-1fc8-4224-9c7d-60ee4fc1a18b","remote_addr":"[fd00:1122:3344:101::c]:45718","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":79,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:57.216898601Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"451bde80-db84-4c3b-a671-17154c04f933","remote_addr":"[fd00:1122:3344:101::c]:45718","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":205,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:57.274246163Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:58794"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:57.274349565Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:57.313942175Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:57.369818149Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"225226ae-feef-4db9-8bcf-12c3e678d5c2","remote_addr":"[fd00:1122:3344:101::b]:58794","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95508,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:57.369995462Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:63541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:57.370024882Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:36542"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:57.370273877Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:57.370347589Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"12276f61-409a-4650-9d8f-cdafb63e906d","remote_addr":"[fd00:1122:3344:101::b]:58794","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":281,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:57.374041679Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"f6a8f7b1-c678-47e6-a231-a7f3c6a9de9a","remote_addr":"[fd00:1122:3344:101::c]:63541","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3762,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:57.404909472Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:57.45269298Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"fa91ac58-ad41-4e08-89a4-4e43f8428781","remote_addr":"[fd00:1122:3344:101::c]:36542","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":82536,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:57.453024676Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"8659cf41-40ce-4112-a487-cb08f5a625f8","remote_addr":"[fd00:1122:3344:101::c]:36542","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":133,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:58.733590469Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56793"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:30:58.733692211Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"7e6cf6fa-d7b7-40f5-aa6c-5f155551608f","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56793","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":35,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:00.544055179Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:41926"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:00.544330424Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"9ef67b83-12f8-4d10-b5f7-029c09775efd","remote_addr":"[fd00:1122:3344:101::a]:41926","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":185,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:00.596894784Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:38575"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:00.597009166Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"69694ce4-3552-4e54-a459-af64b3705595","remote_addr":"[fd00:1122:3344:101::a]:38575","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":55,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:00.640575763Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:00.640597893Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"9f782c16-a65f-4c2e-94b8-d8062d1572b4","remote_addr":"[fd00:1122:3344:101::a]:38575","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":83,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:00.649024125Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"0976f2d4-e597-4d7f-b648-cd64b0d88c5c","remote_addr":"[fd00:1122:3344:101::a]:38575","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":218,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:00.718803605Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:34660"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:00.718895106Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:00.757441487Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:00.817624172Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"79f2fe72-380d-4882-af2d-511c4c24e2b7","remote_addr":"[fd00:1122:3344:101::a]:34660","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98758,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:00.817975719Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"2e0acb1b-b717-43f9-bbda-81c9d9626b96","remote_addr":"[fd00:1122:3344:101::a]:34660","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":117,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:05.736355836Z","hostname":"buskin","pid":1566,"uri":"/198593b5-c3c6-4ef5-879a-52095a2ef511","method":"GET","req_id":"c43374e0-7aad-4e99-ba2b-7bdbf10399a0","remote_addr":"[fd00:1122:3344:101::d]:59260","local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7532,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:27.068299532Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:45140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:27.068449895Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"5399668a-a86d-4860-bb02-c31a78eafb4a","remote_addr":"[fd00:1122:3344:101::b]:45140","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":83,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:27.12287792Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:27.122917451Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"5fd4ad3c-c45e-46bc-aa1f-3d84174d492f","remote_addr":"[fd00:1122:3344:101::b]:45140","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:27.204781523Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:57235"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:27.205005537Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"73d1b452-9306-49e0-bb2a-d0138a7e9ec2","remote_addr":"[fd00:1122:3344:101::b]:57235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":149,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:27.216926977Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"ecbbd934-711a-473c-8173-791197ce346b","remote_addr":"[fd00:1122:3344:101::b]:45140","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":272,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:27.321616837Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:52493"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:27.321858222Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"7e15473e-c796-4823-8146-18af47a93536","remote_addr":"[fd00:1122:3344:101::c]:52493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":163,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:27.384676448Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41706"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:27.384814091Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:27.438270148Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:27.516990519Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"bd00a89a-96a8-4127-a8d6-110b8d8dbf74","remote_addr":"[fd00:1122:3344:101::b]:41706","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":132227,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:27.517130732Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:55557"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:27.517313235Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"0e2bc94c-5711-4c7f-acc6-746163227e4c","remote_addr":"[fd00:1122:3344:101::c]:55557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":87,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:27.517430198Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"6f74ecc5-2dfc-4fc3-b09c-63024ddc8687","remote_addr":"[fd00:1122:3344:101::b]:41706","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":152,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:27.5352554Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:44829"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:27.535440134Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:27.577697575Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:27.636485234Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"ad242ccf-1f52-4c01-91d6-235c51ac1bc4","remote_addr":"[fd00:1122:3344:101::c]:44829","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101131,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:27.636757569Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:27.63677862Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"5a12d1b6-007c-4b15-8279-49926081f605","remote_addr":"[fd00:1122:3344:101::c]:55557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:27.636941413Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"c181819a-297e-4278-8d9d-9d1d907bdb0d","remote_addr":"[fd00:1122:3344:101::c]:44829","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":190,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:27.646461186Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"d65e3c4c-cdf1-488d-99eb-4e41645c3297","remote_addr":"[fd00:1122:3344:101::c]:55557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":240,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:28.768510884Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48294"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:28.768628386Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"ebb84ed4-478b-4e8e-8bbd-9216c6467c52","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48294","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":42,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:30.705702757Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:59268"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:30.70585731Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"6f1011d7-edda-4c33-8629-a738020f4674","remote_addr":"[fd00:1122:3344:101::a]:59268","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":60,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:30.753930284Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:30.753971634Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"2bfa105a-7921-4853-b3f9-31986c316dd4","remote_addr":"[fd00:1122:3344:101::a]:59268","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":69,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:30.763967906Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"2a9eddf0-73e0-4b53-861d-1fc424896971","remote_addr":"[fd00:1122:3344:101::a]:59268","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":209,"response_code":"204"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:30.82194888Z","hostname":"buskin","pid":1566,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:197"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:30.822606403Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:30.861706063Z","hostname":"buskin","pid":1566,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.9999939s","next_cleanup":"Instant { tv_sec: 1999, tv_nsec: 176998903 }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:30.886561801Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:42268"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:30.886807105Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"e26e5e3a-fa62-4851-81cc-efcf9c9a1a36","remote_addr":"[fd00:1122:3344:101::a]:42268","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":161,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:30.921954381Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:60608"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:30.922072983Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:30.957501903Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:31.013638952Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"c5060e40-26fc-4399-af51-b8b446e9a0ac","remote_addr":"[fd00:1122:3344:101::a]:60608","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91628,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:31.013922597Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"9da88e98-3559-44d7-867a-285cf1a7a989","remote_addr":"[fd00:1122:3344:101::a]:60608","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":124,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:35.736661076Z","hostname":"buskin","pid":1566,"uri":"/198593b5-c3c6-4ef5-879a-52095a2ef511","method":"GET","req_id":"88dd754e-fbb1-412f-8847-2056cdb276a3","remote_addr":"[fd00:1122:3344:101::d]:59260","local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8343,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:44.662821872Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:57.031724674Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:62730"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:57.031881417Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"0e19be23-070d-4812-aefd-093c9fa70c2d","remote_addr":"[fd00:1122:3344:101::b]:62730","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":69,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:57.090476192Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:57.090520933Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"a11e5ebe-4791-4b48-9ffa-3bf8dea60b09","remote_addr":"[fd00:1122:3344:101::b]:62730","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":84,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:57.100732669Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"46b52f0f-bb7e-4ccf-b98c-fa993a5353ab","remote_addr":"[fd00:1122:3344:101::b]:62730","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":218,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:57.141659245Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:43268"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:57.141752207Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:57.179552843Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:57.23199844Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"845e972f-768a-4cca-a31b-6368f4e8d822","remote_addr":"[fd00:1122:3344:101::b]:43268","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":90281,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:57.232132873Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:60869"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:57.232182824Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:60445"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:57.232385568Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"afaa69f9-1835-4565-b073-c8febe8a05ca","remote_addr":"[fd00:1122:3344:101::c]:60445","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:57.232604662Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"b37489a0-e937-43c0-9f03-30484a01a0c7","remote_addr":"[fd00:1122:3344:101::b]:60869","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":217,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:57.232613852Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"72240469-4e6a-44c4-99c7-defeeb1d6801","remote_addr":"[fd00:1122:3344:101::b]:43268","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":265,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:57.258167703Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:35717"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:57.258368077Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"0d8d1e23-c8e5-472a-8a68-c26803b6c6b8","remote_addr":"[fd00:1122:3344:101::c]:35717","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":138,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:57.279658826Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:57.279681336Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"bc8a2371-cef4-4bc9-9171-8d75fa8471c2","remote_addr":"[fd00:1122:3344:101::c]:60445","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":68,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:57.288087028Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"b4b743bc-c66d-4f38-b0f7-44306521d542","remote_addr":"[fd00:1122:3344:101::c]:60445","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":219,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:57.321715053Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:48788"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:57.321805635Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:57.354005303Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:57.406979471Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"57d385db-0a87-4f62-b32e-6f28b91f12e9","remote_addr":"[fd00:1122:3344:101::c]:48788","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":85199,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:57.407342108Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"9c73fa51-a5c0-4957-997c-0892ad1547f7","remote_addr":"[fd00:1122:3344:101::c]:48788","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":162,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:58.801567523Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38292"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:31:58.801727166Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"62551cdb-40b3-4eb9-a936-d138c36142da","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38292","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":77,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:00.600496352Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:39790"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:00.600733897Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"9620f3b8-c4a1-45b8-8b28-f7daab89bebc","remote_addr":"[fd00:1122:3344:101::a]:39790","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":137,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:00.65973173Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:00.65976334Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"a076d2e7-218a-4160-b50e-70560002afd1","remote_addr":"[fd00:1122:3344:101::a]:39790","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":121,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:00.668275994Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:47408"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:00.668530469Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"5bec3b6f-0281-47dc-8068-1a1686a75439","remote_addr":"[fd00:1122:3344:101::a]:47408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":175,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:00.680646851Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"eb6eb117-59df-425e-99da-3d82b3045bd9","remote_addr":"[fd00:1122:3344:101::a]:39790","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":297,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:00.746926144Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:33820"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:00.747067616Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:00.803326517Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:00.858590698Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"0674fdf0-0358-4e41-a405-eeeac0741159","remote_addr":"[fd00:1122:3344:101::a]:33820","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":111587,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:00.858899944Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"ee6d6ed5-5048-4c77-95b9-2b473aa41630","remote_addr":"[fd00:1122:3344:101::a]:33820","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":108,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:05.735898996Z","hostname":"buskin","pid":1566,"uri":"/198593b5-c3c6-4ef5-879a-52095a2ef511","method":"GET","req_id":"35287441-54b2-44da-803e-5a90896ce5bf","remote_addr":"[fd00:1122:3344:101::d]:59260","local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7268,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:27.26923412Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:52218"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:27.269428994Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"9eccbde7-773f-4680-87bd-9b04c697e850","remote_addr":"[fd00:1122:3344:101::b]:52218","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":106,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:27.314976898Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:27.315000919Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"04403ce9-7146-4b45-a206-934c8717d3be","remote_addr":"[fd00:1122:3344:101::b]:52218","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":85,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:27.328464757Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:62035"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:27.328675981Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"a067400a-05f9-4c68-a1a2-8ed257125f8f","remote_addr":"[fd00:1122:3344:101::b]:62035","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":148,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:27.33692019Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"e25e352e-ff0c-4e1c-8a30-f0a5425a21f8","remote_addr":"[fd00:1122:3344:101::b]:52218","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":237,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:27.358579726Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:49598"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:27.358679168Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:27.396133197Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:27.457124868Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"b5ddd14c-73f3-4850-949e-8ea45aedc17e","remote_addr":"[fd00:1122:3344:101::b]:49598","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98481,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:27.457348043Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:61059"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:27.457422114Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"afc9fc04-2a15-4708-89b2-d0892d811bec","remote_addr":"[fd00:1122:3344:101::b]:49598","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":129,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:27.457516666Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"1d22e3fe-2cb8-4d08-be74-67b9f5b2e9ed","remote_addr":"[fd00:1122:3344:101::c]:61059","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":72,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:27.501135344Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:27.501165884Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"edd7e587-3a55-42d7-8803-2c2c75264c34","remote_addr":"[fd00:1122:3344:101::c]:61059","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":109,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:27.510039915Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"dc789356-80fb-402d-a16e-c56415dc6199","remote_addr":"[fd00:1122:3344:101::c]:61059","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":309,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:27.557808822Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:36452"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:27.557966815Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"7d863ac1-8ffd-4bd0-b129-2fabfae6199f","remote_addr":"[fd00:1122:3344:101::c]:36452","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":117,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:27.572876251Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:38069"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:27.572986873Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:27.613667675Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:27.671849592Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"c02e8bf3-492a-45c7-8ec8-ce6bd1302f22","remote_addr":"[fd00:1122:3344:101::c]:38069","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98913,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:27.674722537Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"4471ba3e-8111-4c56-895a-dee3a21eacaf","remote_addr":"[fd00:1122:3344:101::c]:38069","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:28.83710199Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33221"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:28.837186792Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"5647c4f8-3f15-411f-9cf0-909771961b32","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33221","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":32,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:30.835466828Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:60493"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:30.835617491Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"9a78ff15-5a9a-4dca-a189-c3dff6a7a8b5","remote_addr":"[fd00:1122:3344:101::a]:60493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:30.85063478Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:30.85065097Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"5f597e5b-e05d-4310-b09b-80109369cb6f","remote_addr":"[fd00:1122:3344:101::a]:60493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":71,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:30.860276865Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"1fa6da48-e0ef-46a2-b515-161e0a6795e1","remote_addr":"[fd00:1122:3344:101::a]:60493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":205,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:30.912377333Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:44821"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:30.912466931Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:30.952510716Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:31.013718162Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"ebb3ab12-6308-4096-90c2-459d437c6ce8","remote_addr":"[fd00:1122:3344:101::a]:44821","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101219,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:31.013900795Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:54359"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:31.014205401Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"c3bcc615-3ccc-45b1-be11-286169cb3867","remote_addr":"[fd00:1122:3344:101::a]:54359","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":218,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:31.014223841Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"a6c15568-77be-4f52-90d3-ec737baf71f2","remote_addr":"[fd00:1122:3344:101::a]:44821","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":241,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:35.736797937Z","hostname":"buskin","pid":1566,"uri":"/198593b5-c3c6-4ef5-879a-52095a2ef511","method":"GET","req_id":"1f3a3185-c188-4d9b-ba2b-c8a6836ff8aa","remote_addr":"[fd00:1122:3344:101::d]:59260","local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7030,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:57.024419903Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:50368"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:57.024580196Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"8598e9e9-601e-490c-a755-fe7d7b5662f8","remote_addr":"[fd00:1122:3344:101::b]:50368","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":66,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:57.088536474Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:57.088568625Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"76d73578-3cc7-431d-ba73-728ffaeb5cce","remote_addr":"[fd00:1122:3344:101::b]:50368","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":81,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:57.089320089Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:40114"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:57.089545833Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"2764c168-c0be-4aec-b1c5-b668ae5183ae","remote_addr":"[fd00:1122:3344:101::b]:40114","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":138,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:57.097376624Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"882f4f75-f627-46c0-ad37-014e4e2e7d20","remote_addr":"[fd00:1122:3344:101::b]:50368","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":240,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:57.17579396Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:53961"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:57.175913502Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:57.212906303Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:57.27834697Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"c9b8d439-2eca-43f4-a722-58d64b6decaf","remote_addr":"[fd00:1122:3344:101::b]:53961","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":102482,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:57.278599794Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:47478"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:57.278711746Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"a5218815-0018-4342-942e-f64cff683640","remote_addr":"[fd00:1122:3344:101::b]:53961","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":160,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:57.278852239Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"375e8bbf-dd19-4385-b02a-21638a6f694e","remote_addr":"[fd00:1122:3344:101::c]:47478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":90,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:57.320921207Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:37505"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:57.321130621Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:57.360056229Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:57.419568512Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"47718912-0ed7-46c8-9561-caec783291da","remote_addr":"[fd00:1122:3344:101::c]:37505","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98536,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:57.419838787Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:63401"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:57.419853697Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:57.419871888Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"6ae56cb3-f455-46dc-b7a1-462f89aa8e89","remote_addr":"[fd00:1122:3344:101::c]:47478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":133,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:57.420050931Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"ecf6b9c1-b06c-4486-bab8-08bdb2c4689e","remote_addr":"[fd00:1122:3344:101::c]:37505","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":241,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:57.420170253Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"4ba61673-35b4-45fc-84b1-42426b4330f7","remote_addr":"[fd00:1122:3344:101::c]:63401","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":195,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:57.429710506Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"07bab054-406e-48e4-8930-5b6bf1898e32","remote_addr":"[fd00:1122:3344:101::c]:47478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":214,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:58.871541286Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36677"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:32:58.871668399Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"45462ebc-76b4-46fc-a293-4acc2e7461fc","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36677","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":74,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:00.655010898Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:54338"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:00.65514173Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"9a4c04b7-d7bd-4638-aec2-44db98d9a575","remote_addr":"[fd00:1122:3344:101::a]:54338","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":55,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:00.6691877Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:00.669208431Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"dd9d1189-f6ba-48e3-850f-3ad7a18b43cb","remote_addr":"[fd00:1122:3344:101::a]:54338","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":64,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:00.67751923Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"34d45131-6a5e-44b6-bc9d-c00acf95edaf","remote_addr":"[fd00:1122:3344:101::a]:54338","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":208,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:00.722556474Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:48960"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:00.722705447Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"e273181e-8892-4dc8-a268-0e0ee5044c44","remote_addr":"[fd00:1122:3344:101::a]:48960","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:00.73641655Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:51416"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:00.736497682Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:00.776657553Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:00.834659117Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"72d63d9d-4275-4948-9675-ff547b151d43","remote_addr":"[fd00:1122:3344:101::a]:51416","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98181,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:00.834947283Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"a30c513d-b158-413f-bbdb-14e60f847e26","remote_addr":"[fd00:1122:3344:101::a]:51416","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":119,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:05.737857873Z","hostname":"buskin","pid":1566,"uri":"/198593b5-c3c6-4ef5-879a-52095a2ef511","method":"GET","req_id":"3114e7b2-708e-4498-9516-9200d04a550e","remote_addr":"[fd00:1122:3344:101::d]:59260","local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7834,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:27.128375114Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:51402"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:27.128540567Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"2d2750f4-ce52-420a-8e52-e3f382bfac3e","remote_addr":"[fd00:1122:3344:101::b]:51402","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":60,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:27.189308124Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:27.189342374Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"2570bacf-a94b-4a03-a047-c4e4725231e1","remote_addr":"[fd00:1122:3344:101::b]:51402","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":86,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:27.200001179Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"5baff89b-3bf6-4da3-b403-e4d248d36508","remote_addr":"[fd00:1122:3344:101::b]:51402","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":246,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:27.259013263Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:48457"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:27.259138865Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"57246507-58a4-41cf-bd7d-1d3d1554aca8","remote_addr":"[fd00:1122:3344:101::c]:48457","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":67,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:27.327794373Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:27.327831464Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"d9e92b33-fbf7-446f-97fc-341bfa2f98a7","remote_addr":"[fd00:1122:3344:101::c]:48457","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:27.339705502Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"93deed87-5cb6-41a4-8776-c3d49768cb26","remote_addr":"[fd00:1122:3344:101::c]:48457","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":319,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:27.362314446Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:53869"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:27.362592262Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"9ffae4d2-bab8-4ff8-8523-4619015a1a44","remote_addr":"[fd00:1122:3344:101::b]:53869","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":206,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:27.417118729Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:33729"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:27.417345373Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:27.459198947Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:27.518286642Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"fac22dfd-80ec-4b00-bc43-92d8250dddac","remote_addr":"[fd00:1122:3344:101::b]:33729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101035,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:27.518562067Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:50138"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:27.518884283Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"5cdb5317-819e-46ce-a2cb-5156028bd12d","remote_addr":"[fd00:1122:3344:101::b]:33729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":281,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:27.518964095Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:27.530444905Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:63324"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:27.530722211Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"9ad3cfd4-0ae0-4cb7-b7fe-e123b7ec39d6","remote_addr":"[fd00:1122:3344:101::c]:63324","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":168,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:27.560613154Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:27.624831268Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"212f6cbd-c93e-4838-b65f-0f67cac1accb","remote_addr":"[fd00:1122:3344:101::c]:50138","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":106012,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:27.625218826Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"b4afd110-6735-47a5-998a-4d8299e99f6e","remote_addr":"[fd00:1122:3344:101::c]:50138","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":150,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:28.909914078Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64291"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:28.9100652Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"b953113c-17ad-4cc6-b503-b380b65870e2","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64291","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":72,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:30.657449069Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:62975"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:30.657611522Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"33a6c924-4ca3-4d9a-92dc-32f1aea0b124","remote_addr":"[fd00:1122:3344:101::a]:62975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:30.68250232Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:30.682535931Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"1bd1dcdc-cd53-420d-94ca-62bbe8794862","remote_addr":"[fd00:1122:3344:101::a]:62975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:30.691267519Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"c394397d-834b-4ecb-90ba-433fbf63d866","remote_addr":"[fd00:1122:3344:101::a]:62975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":221,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:30.850796972Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:34765"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:30.850974745Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"1671f011-6403-4888-81ea-e9119233a663","remote_addr":"[fd00:1122:3344:101::a]:34765","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":143,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:30.960462048Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:42370"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:30.9605637Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:31.000452596Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:31.055161347Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"dad87df5-d967-4700-8db5-6290103e243e","remote_addr":"[fd00:1122:3344:101::a]:42370","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94651,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:31.055401441Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"1833d02b-e4c5-41be-8ce8-e5102a6faa63","remote_addr":"[fd00:1122:3344:101::a]:42370","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":105,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:35.73696715Z","hostname":"buskin","pid":1566,"uri":"/198593b5-c3c6-4ef5-879a-52095a2ef511","method":"GET","req_id":"f7d0fec7-09c0-4af5-b430-e6bc0a520387","remote_addr":"[fd00:1122:3344:101::d]:59260","local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7595,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:56.949854739Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:61752"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:56.949987302Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"bf0ffde0-af68-447e-8157-7a5d87726729","remote_addr":"[fd00:1122:3344:101::b]:61752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":62,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:56.99879359Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:56.99882362Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"f9698ba8-eb9f-4fa4-b916-ed98239ba9ed","remote_addr":"[fd00:1122:3344:101::b]:61752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":81,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:57.007756971Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"82d16822-f7f7-4050-be1f-fe15c8446050","remote_addr":"[fd00:1122:3344:101::b]:61752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":195,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:57.142779265Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:39787"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:57.142949478Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"672e4ba8-b597-47b0-88fd-4747de8b38cd","remote_addr":"[fd00:1122:3344:101::b]:39787","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":119,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:57.191251906Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:34984"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:57.191327697Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"d04e231f-9e2c-44d0-8703-cb07de72e7ba","remote_addr":"[fd00:1122:3344:101::c]:34984","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":24,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:57.200796829Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:53711"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:57.201003453Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:57.240604994Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:57.300722578Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"7c89a41b-00b2-4e03-bf32-3ad9cc1fbba2","remote_addr":"[fd00:1122:3344:101::b]:53711","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99783,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:57.300964083Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:52818"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:57.301207698Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:57.301249198Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"9b8fae9d-a9ab-43ac-a5ee-65a62d1a9675","remote_addr":"[fd00:1122:3344:101::c]:34984","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":209,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:57.30132858Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:57.301430412Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"ad79d08c-d307-421f-a58a-859eddbbc49b","remote_addr":"[fd00:1122:3344:101::b]:53711","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":331,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:57.335587528Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:45130"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:57.335856443Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"d5be643b-2bd0-4aca-aba7-9eb11424382b","remote_addr":"[fd00:1122:3344:101::c]:45130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":162,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:57.339123656Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"b45e8c99-195f-4469-bb91-3dc8a0b91110","remote_addr":"[fd00:1122:3344:101::c]:34984","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":239,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:57.340777518Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:57.398006966Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"9b604723-3a09-4455-925f-84e0cf2f28a3","remote_addr":"[fd00:1122:3344:101::c]:52818","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96855,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:57.398309512Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"64181a88-2b07-434d-8fc3-7d8f30042774","remote_addr":"[fd00:1122:3344:101::c]:52818","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":128,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:58.945661069Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40593"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:33:58.945765891Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"0313a32f-12cf-4e80-a7b4-be69e6daf890","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40593","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":29,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:00.560635794Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:61659"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:00.560871749Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"7135c77b-3509-4ea0-bee0-19a18516e5b4","remote_addr":"[fd00:1122:3344:101::a]:61659","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":172,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:00.595457273Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:33700"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:00.595576826Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"191fd0d5-bf15-4e33-91fa-ac1319b49f76","remote_addr":"[fd00:1122:3344:101::a]:33700","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":78,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:00.609552564Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:00.609570604Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"f34719d1-e04a-4f30-9a69-fccda23618c5","remote_addr":"[fd00:1122:3344:101::a]:33700","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":70,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:00.620168028Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"7187f893-f736-4a2a-843a-6b93b2cc5715","remote_addr":"[fd00:1122:3344:101::a]:33700","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":277,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:00.731998285Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:46122"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:00.732103037Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:00.768307272Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:00.823252967Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"a9529762-a0b4-47a9-ac66-cea5bb06e522","remote_addr":"[fd00:1122:3344:101::a]:46122","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91186,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:00.823550953Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"db545e01-6505-4fcd-b5df-510703b767fe","remote_addr":"[fd00:1122:3344:101::a]:46122","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":122,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:05.73727824Z","hostname":"buskin","pid":1566,"uri":"/198593b5-c3c6-4ef5-879a-52095a2ef511","method":"GET","req_id":"44e335e2-1811-4374-ae96-2c225008ba73","remote_addr":"[fd00:1122:3344:101::d]:59260","local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7514,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.202470496Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:46048"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.202596988Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"6e912dfe-5236-4586-8d8b-1752f9c817a3","remote_addr":"[fd00:1122:3344:101::b]:46048","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":52,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.227593808Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.227625119Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"73a8f0ef-4f41-4162-89c3-d849f75e7c8e","remote_addr":"[fd00:1122:3344:101::b]:46048","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":75,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.350609571Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:44175"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.350710593Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"a005a8df-63e6-45ff-8bc0-105031700f42","remote_addr":"[fd00:1122:3344:101::c]:44175","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":36,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.400678882Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.400719913Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"308115eb-270d-4a2a-bd97-6a5875c3956d","remote_addr":"[fd00:1122:3344:101::c]:44175","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":67,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.410228026Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"6c2fa999-8989-4461-940d-b0e671a5b734","remote_addr":"[fd00:1122:3344:101::c]:44175","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":307,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.430409303Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:42950"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.430599377Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"8e0b942f-7901-4d6b-81a8-315d85b298ec","remote_addr":"[fd00:1122:3344:101::b]:42950","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":138,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.446282018Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"c9a7dfa2-ad65-42f5-8aea-fc5e744d27a5","remote_addr":"[fd00:1122:3344:101::b]:46048","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":288,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.513694403Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:36771"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.513839666Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"a592f7cc-9ca0-4b49-bd2f-b986cf9e7f5c","remote_addr":"[fd00:1122:3344:101::c]:36771","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":106,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.529516607Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:48265"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.529652719Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.529662439Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.5297026Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.530242401Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.536036252Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:36036"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.53643048Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.53646025Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.536562822Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.537078772Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.546688046Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.546757478Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.551282665Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:52158"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.551720593Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.558670247Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:53183"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.562242335Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.562276056Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.591091579Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.591197471Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.601886067Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.608134127Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.608218918Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.624071593Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.624137594Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_736d5541-d5bd-460c-a5f5-441d6996aa03"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.640239193Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.640473938Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_736d5541-d5bd-460c-a5f5-441d6996aa03"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.653793463Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.653845444Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_65ae8c88-c7ba-40ff-af14-085d448fc3c0"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.670116607Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.670694308Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:50705"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.679762852Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.680001187Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_9764f33b-5a77-475e-83b5-777615774b6a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.680976286Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"df1d4032-abed-415b-82ac-150e2bfbc834","remote_addr":"[fd00:1122:3344:101::b]:52158","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":129410,"response_code":"204"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.685157866Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.685248338Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.685310359Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.685336529Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.68537445Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"5ec5c103-582d-4a02-a4af-acf2734534c8","remote_addr":"[fd00:1122:3344:101::b]:52158","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":381,"response_code":"200"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.685410371Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.685435011Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.685475032Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.685578254Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"a3d0ff65-5c4a-4943-8853-3153aafd3804","remote_addr":"[fd00:1122:3344:101::b]:50705","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4905,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.696375621Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.696445173Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_65ae8c88-c7ba-40ff-af14-085d448fc3c0"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.714838055Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.71818192Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.718463145Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_28fb7d98-1494-4570-b25a-aa56857fbe19"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.733214228Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.73329374Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_9764f33b-5a77-475e-83b5-777615774b6a"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.758782359Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.758871481Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_28fb7d98-1494-4570-b25a-aa56857fbe19"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.764635822Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:45558"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.764954218Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.765012909Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.76505248Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.76506719Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.765098431Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.765120341Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.765142632Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.765166412Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"d0126b51-e31b-4bf9-9465-d0876b55b6b1","remote_addr":"[fd00:1122:3344:101::c]:45558","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":428,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.772133576Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.772239098Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_723d2041-12e1-48e9-aa34-7148357fe80c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.780424435Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"5269cef7-3f9d-4c41-8b37-03a836d90a4e","remote_addr":"[fd00:1122:3344:101::c]:53183","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":221616,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.780935315Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"39599b22-50a5-48da-8521-fb9e6c9974c8","remote_addr":"[fd00:1122:3344:101::c]:53183","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":280,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.790258894Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.790502668Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_723d2041-12e1-48e9-aa34-7148357fe80c"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.805740991Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.805796912Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_7980b762-6ad3-435b-bf8d-91d18dcf75d4"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.834658377Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.834741998Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.849885589Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.8499448Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_7980b762-6ad3-435b-bf8d-91d18dcf75d4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.854404386Z","hostname":"buskin","pid":1566,"uri":"/inventory","method":"GET","req_id":"2714c9b6-bc79-43b7-933a-799a2b67b04f","remote_addr":"[fd00:1122:3344:101::c]:36036","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":318112,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.879514998Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.879581809Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:27.896906222Z","hostname":"buskin","pid":1566,"uri":"/inventory","method":"GET","req_id":"c8f2ae92-06ff-4b55-86ed-5fcf9c27cde1","remote_addr":"[fd00:1122:3344:101::b]:48265","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":367255,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:28.979527903Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34027"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:28.979788728Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"63b55157-fda0-405e-9af3-3789aaac80b8","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34027","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":115,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:30.576941652Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:54738"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:30.577071175Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"9048e55e-e42d-48e5-b11e-4d516d11df54","remote_addr":"[fd00:1122:3344:101::a]:54738","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":52,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:30.6419341Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:30.641970011Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"45ec6561-ad43-49ae-b38d-31e1a88ed5c4","remote_addr":"[fd00:1122:3344:101::a]:54738","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":86,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:30.651517684Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"12d23416-76c4-4a17-ba0a-9e720db78e8f","remote_addr":"[fd00:1122:3344:101::a]:54738","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":225,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:30.758317535Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:56746"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:30.758550239Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"b025af3c-cb8d-4e25-a14e-ea13f93bce58","remote_addr":"[fd00:1122:3344:101::a]:56746","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":179,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:30.833341596Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:65124"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:30.833445418Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:30.878003343Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:30.945613082Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"9619e6c2-e8f0-4184-87f5-414a2a5293b6","remote_addr":"[fd00:1122:3344:101::a]:65124","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":112202,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:30.945794476Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:64482"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:30.945965929Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:30.94600192Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/77444133-e801-4aba-9330-b96b8b00119b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:30.946119692Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:30.946130042Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"0b62ed18-bd55-4652-a130-778a4d3f5f97","remote_addr":"[fd00:1122:3344:101::a]:65124","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":253,"response_code":"200"} {"msg":"Reading ledger from /pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:30.946408507Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 47f34c1f-e641-403f-804d-b2c26f2cb1f9 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:30.959004109Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:30.95906506Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_47f34c1f-e641-403f-804d-b2c26f2cb1f9"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 5db46ce3-4889-4034-8561-7e871bb98ae0 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:30.982791896Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:30.982849817Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_5db46ce3-4889-4034-8561-7e871bb98ae0"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 736d5541-d5bd-460c-a5f5-441d6996aa03 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:31.007752455Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:31.007816827Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_736d5541-d5bd-460c-a5f5-441d6996aa03"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 65ae8c88-c7ba-40ff-af14-085d448fc3c0 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:31.03245808Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:31.032518441Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_65ae8c88-c7ba-40ff-af14-085d448fc3c0"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 9764f33b-5a77-475e-83b5-777615774b6a (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:31.051958035Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:31.052024666Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_9764f33b-5a77-475e-83b5-777615774b6a"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:31.055005543Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:38771"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:31.055470372Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a9131c91-4ca4-491f-8265-8e221e38b9ad, kind: Service { id: dc5f3810-93b6-44f1-815c-44411abf99ad }, name: Name(\"external-dns-dc5f3810-93b6-44f1-815c-44411abf99ad\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 181, 71])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: a95ae8ff-aa26-4675-97b6-6b8d75e9ff0e, kind: Service { id: f4635fcc-61fc-4ff9-8203-b560522a972b }, name: Name(\"external-dns-f4635fcc-61fc-4ff9-8203-b560522a972b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 37bada54-3ac8-4367-9487-f47381e04cf6, kind: Service { id: bda5150c-3889-413e-a774-9d40d8f8536d }, name: Name(\"nexus-bda5150c-3889-413e-a774-9d40d8f8536d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 5b4d3cb4-34c8-4838-94e3-102c732f938c, kind: Service { id: 1d7f4b38-ce97-4d7c-b37c-92212a02a45f }, name: Name(\"nexus-1d7f4b38-ce97-4d7c-b37c-92212a02a45f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 159])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: b50b269f-d32a-4af8-be28-7e03354f4d63, kind: Service { id: 1b5ada24-1ce5-4a0b-9c11-3c803e3e9270 }, name: Name(\"nexus-1b5ada24-1ce5-4a0b-9c11-3c803e3e9270\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:31.055523863Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:31.055590874Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:31.055600024Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:31.055636205Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:31.055678056Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:31.055713687Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:31.055734587Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"4d69ca57-ca43-4815-9e9c-aecca9a14476","remote_addr":"[fd00:1122:3344:101::a]:38771","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":565,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 28fb7d98-1494-4570-b25a-aa56857fbe19 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:31.075444475Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:31.075507717Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_28fb7d98-1494-4570-b25a-aa56857fbe19"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 723d2041-12e1-48e9-aa34-7148357fe80c (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:31.099308934Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:31.099369755Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_723d2041-12e1-48e9-aa34-7148357fe80c"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 7980b762-6ad3-435b-bf8d-91d18dcf75d4 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:31.122938848Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:31.122983979Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_7980b762-6ad3-435b-bf8d-91d18dcf75d4"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 3e6fa380-13d2-4109-81e5-c336fd5adffa (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:31.146330847Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:31.146380518Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_3e6fa380-13d2-4109-81e5-c336fd5adffa"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:31.160345926Z","hostname":"buskin","pid":1566,"uri":"/inventory","method":"GET","req_id":"56c19061-474d-40b1-919f-e82d101bad71","remote_addr":"[fd00:1122:3344:101::a]:64482","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":214464,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:35.737611221Z","hostname":"buskin","pid":1566,"uri":"/198593b5-c3c6-4ef5-879a-52095a2ef511","method":"GET","req_id":"780e77da-9b22-4127-9e36-e5a2bb4c9ac0","remote_addr":"[fd00:1122:3344:101::d]:59260","local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7156,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:57.002085432Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:54869"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:57.002194374Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"092c869c-360d-48d5-b6b7-209c9e38b3fd","remote_addr":"[fd00:1122:3344:101::b]:54869","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":50,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:57.063478851Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:57.063522502Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"ad81c361-683c-4335-b146-91d7ce8d8c95","remote_addr":"[fd00:1122:3344:101::b]:54869","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":87,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:57.077780036Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:65411"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:57.0779681Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"633896fc-be78-4dc8-a070-c287f62bc904","remote_addr":"[fd00:1122:3344:101::b]:65411","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":133,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:57.088023583Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"77912b8f-cf9a-40da-8071-b5186cacc662","remote_addr":"[fd00:1122:3344:101::b]:54869","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":181,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:57.158071938Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:43712"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:57.1581646Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:57.2003345Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:57.26389325Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"9b7a7842-1122-48fb-9598-95964ce3b83b","remote_addr":"[fd00:1122:3344:101::b]:43712","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":105770,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:57.264067804Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:36384"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:57.264244797Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"3318599c-9138-4881-8c55-d3d41bdbf7ab","remote_addr":"[fd00:1122:3344:101::c]:36384","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":106,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:57.2643745Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"3c3e9dfb-537b-4b8f-aeb7-6b42ada3af87","remote_addr":"[fd00:1122:3344:101::b]:43712","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":214,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:57.304973609Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:57.30499354Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"ee700092-700d-454e-9045-df47c5299186","remote_addr":"[fd00:1122:3344:101::c]:36384","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":81,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:57.337839911Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:39031"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:57.338071015Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"55ab49b1-fff3-45fb-b08f-acba911cdee2","remote_addr":"[fd00:1122:3344:101::c]:39031","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":161,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:57.341356618Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"af3dda00-fb5c-4437-8fcf-79a3caf83c17","remote_addr":"[fd00:1122:3344:101::c]:36384","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":236,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:57.40030898Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:41906"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:57.400419242Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:57.439632226Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:57.494747494Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"8ad7032b-7f44-4612-9a98-04beb2fcf8fd","remote_addr":"[fd00:1122:3344:101::c]:41906","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94370,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:57.495032679Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"851d4cc8-9801-49b6-b7b2-be8ff5a64110","remote_addr":"[fd00:1122:3344:101::c]:41906","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:59.013091523Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37794"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:34:59.013203045Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"fa1a6729-acea-4f70-a3b9-b5f23975d295","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37794","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":41,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:00.627037269Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:43528"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:00.627160862Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"a74d84e5-001e-4a42-922b-3537f53a2916","remote_addr":"[fd00:1122:3344:101::a]:43528","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":53,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:00.667902194Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:00.667955585Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"766a5a62-9279-42c2-b10f-68fa9ef79b60","remote_addr":"[fd00:1122:3344:101::a]:43528","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":111,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:00.673734846Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:52753"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:00.673892119Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"7123058f-96b3-428e-8e11-fc891b7b8d73","remote_addr":"[fd00:1122:3344:101::a]:52753","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":109,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:00.680281872Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"575eb497-0bd3-4fcb-8e75-7957cd5b6840","remote_addr":"[fd00:1122:3344:101::a]:43528","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":159,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:00.74166879Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:44826"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:00.741760572Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:00.780986465Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:00.839227304Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"992e70c1-b23c-4c58-b9df-537b0a994498","remote_addr":"[fd00:1122:3344:101::a]:44826","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97501,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:00.839484159Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"36a6483b-7d70-49b1-ba82-320048b98125","remote_addr":"[fd00:1122:3344:101::a]:44826","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":92,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:05.73833953Z","hostname":"buskin","pid":1566,"uri":"/198593b5-c3c6-4ef5-879a-52095a2ef511","method":"GET","req_id":"ce8b735e-911f-498e-b430-fa41fdddc5f5","remote_addr":"[fd00:1122:3344:101::d]:59260","local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7823,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:27.095606443Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:50587"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:27.095734826Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"3bc4096b-a484-4f62-9c26-9485dea87c7d","remote_addr":"[fd00:1122:3344:101::b]:50587","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":58,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:27.120315838Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:27.120349038Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"6e642b55-2269-4f39-93ec-8169e3f30cf0","remote_addr":"[fd00:1122:3344:101::b]:50587","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":82,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:27.222852037Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:45666"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:27.223063221Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"63d5fe83-5511-4780-98fb-04d61af6892c","remote_addr":"[fd00:1122:3344:101::b]:45666","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":145,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:27.23187327Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"c91ba804-522d-4674-b34e-e256ef01a57f","remote_addr":"[fd00:1122:3344:101::b]:50587","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":234,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:27.276345134Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:45560"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:27.276452117Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:27.308668495Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:27.364560169Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"53ea23b1-00db-4d30-9edd-aa4b3d21a4f5","remote_addr":"[fd00:1122:3344:101::b]:45560","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88145,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:27.364862944Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"911fca71-09c7-4522-a0d9-d10b228d9951","remote_addr":"[fd00:1122:3344:101::b]:45560","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":131,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:27.374184843Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:50872"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:27.374370667Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"d75c2465-4887-420d-b5ec-7bb2c3b131f5","remote_addr":"[fd00:1122:3344:101::c]:50872","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":80,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:27.415461086Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:36336"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:27.41564206Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"497c78d1-2225-4573-bed5-0c4d70fdfcce","remote_addr":"[fd00:1122:3344:101::c]:36336","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":135,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:27.461016231Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:27.461075622Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"e95b57e7-0b62-4ea2-9be0-8a53b7d05de1","remote_addr":"[fd00:1122:3344:101::c]:50872","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":123,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:27.472039753Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"baf59d88-eafa-47f0-b1a5-b751a1b64963","remote_addr":"[fd00:1122:3344:101::c]:50872","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":197,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:27.519850921Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:51852"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:27.519949043Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:27.557681079Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:27.608222528Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"cbe8fd8d-0453-4d7d-86ec-0341b2304001","remote_addr":"[fd00:1122:3344:101::c]:51852","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88310,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:27.608528894Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"70b7f3c2-e948-4bff-b6d8-014d1fa75491","remote_addr":"[fd00:1122:3344:101::c]:51852","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":113,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:29.047465668Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44582"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:29.04756841Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"80685e58-fa53-4eeb-8eee-f6b2862059e0","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44582","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":31,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:30.685030558Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:38061"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:30.685205621Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"fe73e33f-83cf-4daf-af89-8750520144ca","remote_addr":"[fd00:1122:3344:101::a]:38061","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:30.727061644Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:30.727090955Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"0bc717fb-241f-4609-acc1-e2b1662132de","remote_addr":"[fd00:1122:3344:101::a]:38061","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:30.735860703Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"4e5ed6c9-4f21-458c-bf97-cf25867d7d9e","remote_addr":"[fd00:1122:3344:101::a]:38061","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":243,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:30.785292003Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50286"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:30.785466456Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"84ebd3f4-a406-4c4d-b325-09f9a5afc537","remote_addr":"[fd00:1122:3344:101::a]:50286","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":137,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:30.803302109Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50306"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:30.803398491Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:30.842053103Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:30.903446572Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"b8b2489e-7fb1-4c87-a959-cedc2a4bfd84","remote_addr":"[fd00:1122:3344:101::a]:50306","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100078,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:30.903745808Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"5032c6b9-31d3-4212-9d61-87e5f6cc662d","remote_addr":"[fd00:1122:3344:101::a]:50306","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":111,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:35.739727152Z","hostname":"buskin","pid":1566,"uri":"/198593b5-c3c6-4ef5-879a-52095a2ef511","method":"GET","req_id":"c2bcc5d0-4fc4-487f-9f94-4db4ec0eed55","remote_addr":"[fd00:1122:3344:101::d]:59260","local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8581,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:56.980316614Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:48649"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:56.980441666Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"6fd21a80-e715-4a61-af17-83f3ba52dd61","remote_addr":"[fd00:1122:3344:101::b]:48649","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":52,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:57.037419251Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:57.037461151Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"b664caed-a8a1-4714-844b-ad5f2773d439","remote_addr":"[fd00:1122:3344:101::b]:48649","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":109,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:57.059032776Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"fd826b00-5019-4b3e-af92-780b42c94afd","remote_addr":"[fd00:1122:3344:101::b]:48649","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":243,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:57.114237846Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:36937"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:57.11445937Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"ea2b8e8f-e5d4-45e1-a5b8-c6fd680a764f","remote_addr":"[fd00:1122:3344:101::b]:36937","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":173,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:57.127201935Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:65068"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:57.127309897Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:57.174163687Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:57.234149309Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"b1fa72d2-c21d-468d-ac4c-bd76777b0f45","remote_addr":"[fd00:1122:3344:101::b]:65068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":106883,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:57.234393294Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:46219"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:57.234622268Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"281671da-9baf-46a1-ae8f-297ba7a81497","remote_addr":"[fd00:1122:3344:101::b]:65068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":196,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:57.2346975Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"6f900fe7-2ed2-4d5c-8126-842ae53e11ce","remote_addr":"[fd00:1122:3344:101::c]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":156,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:57.285186649Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:57.285211999Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"95cbc21c-a8e9-4af3-ba67-1b0894e48d95","remote_addr":"[fd00:1122:3344:101::c]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":126,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:57.348470614Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:43989"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:57.348679509Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"2fe8fa1a-f54a-4316-b475-f47016ca5f97","remote_addr":"[fd00:1122:3344:101::c]:43989","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":152,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:57.352321508Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"814202df-ae36-4655-a14d-04a82382439a","remote_addr":"[fd00:1122:3344:101::c]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":273,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:57.399629827Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:44961"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:57.399725579Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:57.439540494Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:57.500559975Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"9fe81701-de25-4f75-9057-56762000bbd4","remote_addr":"[fd00:1122:3344:101::c]:44961","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100867,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:57.500894702Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"9dd295d0-753c-42ac-804d-a55519e4d1a0","remote_addr":"[fd00:1122:3344:101::c]:44961","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:59.081975336Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60755"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:35:59.082126439Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"50f7a411-99ac-4625-aa22-6a1432fdf6b7","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60755","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":44,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:00.583615125Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:34343"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:00.583796318Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"695aec86-bc33-4ac2-b04d-6650e834e573","remote_addr":"[fd00:1122:3344:101::a]:34343","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:00.624046601Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:00.624070962Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"547e9b17-4524-42ba-b8bb-8137e1792a31","remote_addr":"[fd00:1122:3344:101::a]:34343","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":90,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:00.62551224Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:39496"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:00.625693313Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"ad0181e6-1afb-4657-b53c-8e1006958af1","remote_addr":"[fd00:1122:3344:101::a]:39496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":131,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:00.634269488Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"0d408491-4d91-49dc-8aa5-21da429a25c7","remote_addr":"[fd00:1122:3344:101::a]:34343","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":174,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:00.680051137Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:48486"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:00.680156999Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:00.719913322Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:00.777581619Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"2c0f2235-b3f6-4759-a0de-17eabb413066","remote_addr":"[fd00:1122:3344:101::a]:48486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97468,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:00.777855975Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"b541c64f-f133-4689-9fc3-2c50bed3212a","remote_addr":"[fd00:1122:3344:101::a]:48486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:05.73889469Z","hostname":"buskin","pid":1566,"uri":"/198593b5-c3c6-4ef5-879a-52095a2ef511","method":"GET","req_id":"1a1b5b0d-3eca-4cb2-8308-eeeed4e890a8","remote_addr":"[fd00:1122:3344:101::d]:59260","local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7582,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:19.999653336Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:36538"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:19.999928742Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"cc9602dc-3e4d-4e7b-9c44-993593662266","remote_addr":"[fd00:1122:3344:101::b]:36538","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":173,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:20.00091909Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:874","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(3443410) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:20.000952191Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"PUT","req_id":"d5ffc5cf-520d-44bc-92f7-48e42ed5fe35","remote_addr":"[fd00:1122:3344:101::b]:36538","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":846,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:20.181466928Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:39433"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:20.186638237Z","hostname":"buskin","pid":1566,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:555","metadata":"InstanceMetadata { silo_id: 53828d51-c7a3-4557-b913-6b5f10d267df, project_id: 14cc1e33-d5c6-4b7a-bf68-a00ccde56a21 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2025-01-16T23:36:20.181236294Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: f365cf8b-ed44-4683-aef3-1e1250e19e35, kind: Instance { id: 7f99f0f0-65b1-4cc7-8391-f8886f9a2d13 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(3443410), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f365cf8b-ed44-4683-aef3-1e1250e19e35, kind: Instance { id: 7f99f0f0-65b1-4cc7-8391-f8886f9a2d13 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(3443410), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f365cf8b-ed44-4683-aef3-1e1250e19e35, kind: Instance { id: 7f99f0f0-65b1-4cc7-8391-f8886f9a2d13 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(3443410), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(3443410))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f365cf8b-ed44-4683-aef3-1e1250e19e35, kind: Instance { id: 7f99f0f0-65b1-4cc7-8391-f8886f9a2d13 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(3443410), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [InstanceDisk { disk_id: 42b9aff8-4be4-4222-b7ea-26957306a4de, slot: 0, read_only: false, vcr_json: .., name: \"disk-3f00b598329\" }], boot_settings: Some(InstanceBootSettings { order: [42b9aff8-4be4-4222-b7ea-26957306a4de] }), cloud_init_bytes: Some(..) }","migration_id":"None","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:20.186689389Z","hostname":"buskin","pid":1566,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:588","migration_id":"None","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:20.186701309Z","hostname":"buskin","pid":1566,"propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"sled-agent/src/instance.rs:1563","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: f365cf8b-ed44-4683-aef3-1e1250e19e35, kind: Instance { id: 7f99f0f0-65b1-4cc7-8391-f8886f9a2d13 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(3443410), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f365cf8b-ed44-4683-aef3-1e1250e19e35, kind: Instance { id: 7f99f0f0-65b1-4cc7-8391-f8886f9a2d13 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(3443410), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f365cf8b-ed44-4683-aef3-1e1250e19e35, kind: Instance { id: 7f99f0f0-65b1-4cc7-8391-f8886f9a2d13 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(3443410), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(3443410))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f365cf8b-ed44-4683-aef3-1e1250e19e35, kind: Instance { id: 7f99f0f0-65b1-4cc7-8391-f8886f9a2d13 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(3443410), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [InstanceDisk { disk_id: 42b9aff8-4be4-4222-b7ea-26957306a4de, slot: 0, read_only: false, vcr_json: .., name: \"disk-3f00b598329\" }], boot_settings: Some(InstanceBootSettings { order: [42b9aff8-4be4-4222-b7ea-26957306a4de] }), cloud_init_bytes: Some(..) }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2025-01-16T23:36:20.181236294Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400, migration_id: None }","migration_id":"None","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:20.186812751Z","hostname":"buskin","pid":1566,"uri":"/vmms/2f241142-413f-4577-99ce-e5f36b8bb539","method":"PUT","req_id":"c8b80c7a-ad60-48f4-a10e-42dfb9f337a3","remote_addr":"[fd00:1122:3344:101::b]:39433","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5246,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:20.277780908Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:59238"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:20.278788167Z","hostname":"buskin","pid":1566,"uri":"/vmms/2f241142-413f-4577-99ce-e5f36b8bb539/state","method":"PUT","req_id":"9e5e9099-e97b-4ffb-a0cd-253af141a058","remote_addr":"[fd00:1122:3344:101::b]:59238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":895,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:20.280195334Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 240, 0, 0]), slot: 0, vni: Vni { inner: 3443410 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:20.280240195Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:20.325906352Z","hostname":"buskin","pid":1566,"instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:20.351479793Z","hostname":"buskin","pid":1566,"instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:20.364756608Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:65448"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:20.364930882Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"75e7b824-d25b-4e03-9aa2-63986076afd1","remote_addr":"[fd00:1122:3344:101::b]:65448","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":82,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:20.3794213Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {f365cf8b-ed44-4683-aef3-1e1250e19e35: {10.151.1.176: {c16b1b31-9220-4ec4-a2ab-aa48b61c67bf}, 10.151.1.175: {c16b1b31-9220-4ec4-a2ab-aa48b61c67bf}}} }"} {"msg":"Profile for oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:22.013302908Z","hostname":"buskin","pid":1566,"instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:22.013533812Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539\") failed: Property not found. retry in 68.008852ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:36:22.20972069Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539\") failed: Property not found. retry in 101.318471ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:36:22.304877898Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539\") failed: Property not found. retry in 262.33663ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:36:22.421273893Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539\") failed: Property not found. retry in 559.874769ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:36:22.702565875Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539\") failed: Property not found. retry in 538.701904ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:36:23.275241804Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539\") failed: Property not found. retry in 1.046981192s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:36:23.824522772Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539\") failed: Property not found. retry in 698.995011ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:36:24.881410879Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539\") failed: Property not found. retry in 1.456704335s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:36:25.59166926Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:27.044522662Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:40389"} {"msg":"wait for service Some(\"oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539\") failed: Property not found. retry in 1.339202988s","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:36:27.064172019Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:27.212559359Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:50941"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:27.212801364Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"50cb3b25-ab76-4cf6-9bbf-bc6a8348a794","remote_addr":"[fd00:1122:3344:101::b]:50941","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":152,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:27.221400039Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:57890"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:27.430321811Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:35277"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:27.430471804Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:27.470058134Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:27.524060101Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"8ca31d8b-dc8b-4e71-aac2-9a1750b29cea","remote_addr":"[fd00:1122:3344:101::b]:35277","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93640,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:27.524190924Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:36421"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:27.524441809Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"c7eb3c0f-2303-43cc-b49d-0a7358147058","remote_addr":"[fd00:1122:3344:101::c]:36421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":110,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:27.524589861Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"180b9c09-124a-4228-9f4c-e37f5dd48d17","remote_addr":"[fd00:1122:3344:101::b]:35277","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":193,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:27.579250311Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {f365cf8b-ed44-4683-aef3-1e1250e19e35: {10.151.1.176: {c16b1b31-9220-4ec4-a2ab-aa48b61c67bf}, 10.151.1.175: {c16b1b31-9220-4ec4-a2ab-aa48b61c67bf}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:27.579296982Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"830d56c9-00ff-4623-ba97-403e828932a1","remote_addr":"[fd00:1122:3344:101::c]:36421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":200,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:27.699214675Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:58289"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:27.69944163Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"52923b56-fd65-48e2-91bf-b9c82cbfec95","remote_addr":"[fd00:1122:3344:101::c]:58289","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":176,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:27.762681334Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:41123"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:27.762878377Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:27.801623602Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:27.857579636Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"27ba5824-bf51-424b-bf4d-d13af59c3577","remote_addr":"[fd00:1122:3344:101::c]:41123","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94810,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:27.858199188Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"26ebcc2e-b150-42af-bac5-0cfd1cef0c6f","remote_addr":"[fd00:1122:3344:101::c]:41123","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":237,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:27.858777179Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"b45ac1ef-dd18-4351-96fb-d7fd2fd9b66f","remote_addr":"[fd00:1122:3344:101::c]:36421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":744,"response_code":"204"} {"msg":"Started propolis in zone: oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:28.434378734Z","hostname":"buskin","pid":1566,"instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"sled-agent/src/instance.rs:2068"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:28.444790264Z","hostname":"buskin","pid":1566,"instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"sled-agent/src/instance.rs:2077"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:28.483592859Z","hostname":"buskin","pid":1566,"instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"sled-agent/src/instance.rs:2101"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:28.622021228Z","hostname":"buskin","pid":1566,"instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"sled-agent/src/instance.rs:1214"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:28.622771022Z","hostname":"buskin","pid":1566,"uri":"/vmms/2f241142-413f-4577-99ce-e5f36b8bb539/state","method":"GET","req_id":"c6b9b699-ba46-4b53-941f-23e5232221cb","remote_addr":"[fd00:1122:3344:101::c]:57890","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":1398876,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:28.622807733Z","hostname":"buskin","pid":1566,"uri":"/vmms/2f241142-413f-4577-99ce-e5f36b8bb539/state","method":"GET","req_id":"6a10c55e-ce1d-4f2c-80ab-2d47af9fe783","remote_addr":"[fd00:1122:3344:101::b]:40389","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":1574857,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Creating), migration_in: None, migration_out: None, time: 2025-01-16T23:36:28.622955576Z }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:28.622984696Z","hostname":"buskin","pid":1566,"instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"sled-agent/src/instance.rs:781"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:28.623004297Z","hostname":"buskin","pid":1566,"instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"sled-agent/src/instance.rs:803","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2025-01-16T23:36:28.622955576Z }","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:28.623013607Z","hostname":"buskin","pid":1566,"instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2025-01-16T23:36:28.622955576Z }, migration_in: None, migration_out: None }","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13"} {"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:28.623806742Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1249"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:28.623849313Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"a404fe0c-2d79-47dd-8886-af44dd0399b1","remote_addr":"[fd00:1122:3344:101::b]:65448","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8244460,"response_code":"204"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_in: None, migration_out: None, time: 2025-01-16T23:36:28.690628235Z }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:28.690652296Z","hostname":"buskin","pid":1566,"instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"sled-agent/src/instance.rs:781"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:28.690693967Z","hostname":"buskin","pid":1566,"instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"sled-agent/src/instance.rs:803","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2025-01-16T23:36:28.690628235Z }","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:28.690702007Z","hostname":"buskin","pid":1566,"instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2025-01-16T23:36:28.690628235Z }, migration_in: None, migration_out: None }","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13"} {"msg":"skipping delta compute for subnet","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:28.702084846Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:486","new_vers":"RouterVersion { router_id: 0c1ad0f6-8977-429b-a0d2-789d54a9f098, version: 3 }","old_vers":"RouterVersion { router_id: 0c1ad0f6-8977-429b-a0d2-789d54a9f098, version: 3 }","subnet":"RouterId { vni: Vni(3443410), kind: System }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:28.702201548Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"456941b9-ddc0-442e-a29a-db5bde4b1cb7","remote_addr":"[fd00:1122:3344:101::b]:65448","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":266,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:28.744683803Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:37653"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:28.745899126Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"0ff5d88b-79f6-4ca2-88c1-542bdd73d8c6","remote_addr":"[fd00:1122:3344:101::b]:37653","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":38,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:28.762911103Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {f365cf8b-ed44-4683-aef3-1e1250e19e35: {10.151.1.175: {c16b1b31-9220-4ec4-a2ab-aa48b61c67bf}, 10.151.1.176: {c16b1b31-9220-4ec4-a2ab-aa48b61c67bf}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:28.762926583Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"9e89cdd7-24a0-47b1-8b58-7530d256167d","remote_addr":"[fd00:1122:3344:101::b]:37653","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":66,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:28.780279727Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"60a1365d-5b08-4d1b-b286-5e229be0958a","remote_addr":"[fd00:1122:3344:101::b]:37653","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":140,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:29.115543446Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36939"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:29.116438393Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"3cbbf7db-e8fe-4934-9da7-23498d29567f","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36939","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":42,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:30.663711588Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:36342"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:30.66383336Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"f83f3b09-f0c6-4b2b-bb8c-4f48ff4e7070","remote_addr":"[fd00:1122:3344:101::a]:36342","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":54,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:30.789528764Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:41727"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:30.792020722Z","hostname":"buskin","pid":1566,"uri":"/vmms/2f241142-413f-4577-99ce-e5f36b8bb539/state","method":"GET","req_id":"efdc42f6-b6b0-4ca7-bd63-a94557267cc0","remote_addr":"[fd00:1122:3344:101::a]:41727","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":108,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:30.839364981Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {f365cf8b-ed44-4683-aef3-1e1250e19e35: {10.151.1.176: {c16b1b31-9220-4ec4-a2ab-aa48b61c67bf}, 10.151.1.175: {c16b1b31-9220-4ec4-a2ab-aa48b61c67bf}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:30.839401921Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"89c0e464-c158-4098-a5b7-9d4795a407c8","remote_addr":"[fd00:1122:3344:101::a]:36342","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":142,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:30.863976913Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"87bdb693-786a-49a1-84f6-50b8ebafc273","remote_addr":"[fd00:1122:3344:101::a]:36342","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":327,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:30.926171718Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50304"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:30.9262686Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:30.965126386Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:31.022651511Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"ec3c7319-bc35-42c8-aef7-9497011367b0","remote_addr":"[fd00:1122:3344:101::a]:50304","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96419,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:31.022926616Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:64226"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:31.023229942Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"a1fc268e-84ec-4ee6-bcda-2fae7ee6908f","remote_addr":"[fd00:1122:3344:101::a]:50304","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":268,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:31.023335304Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"2c34a017-fb90-42e6-a936-da91274fa1da","remote_addr":"[fd00:1122:3344:101::a]:64226","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":228,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:35.738887595Z","hostname":"buskin","pid":1566,"uri":"/198593b5-c3c6-4ef5-879a-52095a2ef511","method":"GET","req_id":"6a945b85-fefd-4024-86b5-25d2818a1299","remote_addr":"[fd00:1122:3344:101::d]:59260","local_addr":"[fd00:1122:3344:101::1]:42407","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7391,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:38.624054535Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:47477"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:38.624637326Z","hostname":"buskin","pid":1566,"uri":"/vmms/2f241142-413f-4577-99ce-e5f36b8bb539/state","method":"PUT","req_id":"ec90acac-9940-4a6b-a041-0876bc48e8bd","remote_addr":"[fd00:1122:3344:101::b]:47477","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":510,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_in: None, migration_out: None, time: 2025-01-16T23:36:38.624773289Z }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:38.624804059Z","hostname":"buskin","pid":1566,"instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"sled-agent/src/instance.rs:781"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:38.62484342Z","hostname":"buskin","pid":1566,"instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"sled-agent/src/instance.rs:803","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2025-01-16T23:36:38.624773289Z }","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:38.624858171Z","hostname":"buskin","pid":1566,"instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2025-01-16T23:36:38.624773289Z }, migration_in: None, migration_out: None }","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_in: None, migration_out: None, time: 2025-01-16T23:36:38.672023957Z }","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:38.672045357Z","hostname":"buskin","pid":1566,"instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"sled-agent/src/instance.rs:781"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:38.672066517Z","hostname":"buskin","pid":1566,"instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"sled-agent/src/instance.rs:803","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2025-01-16T23:36:38.672023957Z }","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:38.672075378Z","hostname":"buskin","pid":1566,"instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"sled-agent/src/instance.rs:817","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:38.672098878Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:36:38.67219281Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:602","n_samples":6,"id":"11664374420176346875"} {"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:36:38.672285522Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:602","n_samples":6,"id":"13867901812198836955"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:38.672372933Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:38.672402744Z","hostname":"buskin","pid":1566,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:339","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/77444133-e801-4aba-9330-b96b8b00119b/debug/bundle/zone\", \"/pool/int/c380568a-31f5-4b69-bfdd-8314990b56fd/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/47f34c1f-e641-403f-804d-b2c26f2cb1f9/crypt/debug\", \"/pool/ext/5db46ce3-4889-4034-8561-7e871bb98ae0/crypt/debug\", \"/pool/ext/736d5541-d5bd-460c-a5f5-441d6996aa03/crypt/debug\", \"/pool/ext/65ae8c88-c7ba-40ff-af14-085d448fc3c0/crypt/debug\", \"/pool/ext/9764f33b-5a77-475e-83b5-777615774b6a/crypt/debug\", \"/pool/ext/28fb7d98-1494-4570-b25a-aa56857fbe19/crypt/debug\", \"/pool/ext/723d2041-12e1-48e9-aa34-7148357fe80c/crypt/debug\", \"/pool/ext/7980b762-6ad3-435b-bf8d-91d18dcf75d4/crypt/debug\", \"/pool/ext/3e6fa380-13d2-4109-81e5-c336fd5adffa/crypt/debug\"] }","zone_name":"oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:39.31318869Z","hostname":"buskin","pid":1566,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1142","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539\", bundle_id: 1bf09258-3046-4e18-8803-f7476ea42011 }, time_created: 2025-01-16T23:36:38.673287561Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_2f241142-413f-4577-99ce-e5f36b8bb539","v":0,"name":"SledAgent","level":40,"time":"2025-01-16T23:36:39.313266571Z","hostname":"buskin","pid":1566,"instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"sled-agent/src/instance.rs:1314"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:41.083464787Z","hostname":"buskin","pid":1566,"instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"illumos-utils/src/zone.rs:279"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:41.100573236Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:1000","id":"RouterId { vni: Vni(3443410), kind: System }"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:41.100621567Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:1000","id":"RouterId { vni: Vni(3443410), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:41.102075675Z","hostname":"buskin","pid":1566,"instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2025-01-16T23:36:41.102044654Z }, migration_in: None, migration_out: None }","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:41.138132927Z","hostname":"buskin","pid":1566,"instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2025-01-16T23:36:41.102044654Z }, migration_in: None, migration_out: None }","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:41.143668964Z","hostname":"buskin","pid":1566,"instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","propolis_id":"2f241142-413f-4577-99ce-e5f36b8bb539","instance_id":"7f99f0f0-65b1-4cc7-8391-f8886f9a2d13","component":"InstanceManager","file":"sled-agent/src/instance.rs:1251"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:41.77254531Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:45105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:41.772721534Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"b952a603-ef30-42f2-9697-57d305f60501","remote_addr":"[fd00:1122:3344:101::c]:45105","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:41.819021433Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:41.819087655Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1249"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:41.819096284Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"d0451733-666f-4a84-9b79-6b416cb8d18b","remote_addr":"[fd00:1122:3344:101::c]:45105","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":169,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:41.828593397Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"1b79e4fa-f726-495f-bd1d-c4830bcfb42f","remote_addr":"[fd00:1122:3344:101::c]:45105","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":344,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:41.858881979Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:60631"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:41.859074052Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"d86569d0-e547-43c6-8d61-cea0371d63ca","remote_addr":"[fd00:1122:3344:101::c]:60631","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":143,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:43.015336278Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:58529"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:43.015552463Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"3bc0a20c-1921-4ab8-bee7-ca78d4b91d6d","remote_addr":"[fd00:1122:3344:101::b]:58529","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":125,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:43.102762477Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:43121"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:43.10287173Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"3813c903-630d-41f7-83d6-064425024104","remote_addr":"[fd00:1122:3344:101::b]:43121","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":45,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:43.116371349Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:43.116387769Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"dcfec7b6-c5ed-430d-abd5-5a4816c18ceb","remote_addr":"[fd00:1122:3344:101::b]:43121","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":58,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:43.123882223Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"6b853c7b-8424-403f-9ed1-132af74ed19b","remote_addr":"[fd00:1122:3344:101::b]:43121","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":243,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:43.727573027Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:62478"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:43.72776851Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"e7e69549-5b45-4467-a902-1e136998ee03","remote_addr":"[fd00:1122:3344:101::b]:62478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:43.741613466Z","hostname":"buskin","pid":1566,"sled_id":"198593b5-c3c6-4ef5-879a-52095a2ef511","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:43.741636397Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"f5456aeb-9338-4d05-b81c-33dc3b02af88","remote_addr":"[fd00:1122:3344:101::b]:62478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":84,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:43.74963856Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"2ca54117-abd5-42d9-819d-818e95b8a21c","remote_addr":"[fd00:1122:3344:101::b]:62478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":241,"response_code":"204"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-16T23:36:44.666647682Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}