[ Sep 7 18:50:31 Enabled. ] [ Sep 7 18:50:31 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Sep 7 18:50:31 Method "start" exited with status 0. ] [ Sep 7 18:50:31 Rereading configuration. ] [ Sep 7 18:50:31 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:31.369101313Z","hostname":"buskin","pid":1562,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:31.369260117Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:31.421143318Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:31.421191749Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:31.533590004Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:31.542389714Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:31.550232477Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.152187938Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:200::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.288490853Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:126"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.288525394Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:138"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.288633587Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:151"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.288778341Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.288791721Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.288807292Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:716"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.291935847Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:623"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:32.291943447Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:765"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.291959047Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:187"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.291963877Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:824"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.291994898Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.29203901Z","hostname":"buskin","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.292141762Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.292177983Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.292193614Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.303386458Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.303413259Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:371","id":"951f74f7-28ed-48ce-8b8e-a82d754c6f9c (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.403138579Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 951f74f7-28ed-48ce-8b8e-a82d754c6f9c (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.814889122Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.814920532Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.814925392Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.814936283Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.814943733Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.814962993Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.814967604Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.82328345Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.82330235Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:371","id":"f677c761-8af0-4ab3-b20f-2436c20df40c (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:32.909775521Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: f677c761-8af0-4ab3-b20f-2436c20df40c (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.307769648Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.307807459Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 951f74f7-28ed-48ce-8b8e-a82d754c6f9c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f677c761-8af0-4ab3-b20f-2436c20df40c (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.30783694Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.30784484Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.307879451Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.307890491Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.307927312Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.307940643Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 951f74f7-28ed-48ce-8b8e-a82d754c6f9c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f677c761-8af0-4ab3-b20f-2436c20df40c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.307960893Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.307965924Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.307970804Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.307987104Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.307991584Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.307995964Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 951f74f7-28ed-48ce-8b8e-a82d754c6f9c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f677c761-8af0-4ab3-b20f-2436c20df40c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308016855Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308021925Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308026395Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308032585Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308037655Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308041846Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 951f74f7-28ed-48ce-8b8e-a82d754c6f9c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f677c761-8af0-4ab3-b20f-2436c20df40c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308060246Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308073386Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308078687Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308083537Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308088427Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308092527Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 951f74f7-28ed-48ce-8b8e-a82d754c6f9c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f677c761-8af0-4ab3-b20f-2436c20df40c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308112387Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308115908Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308120708Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308124988Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308131628Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308135338Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 951f74f7-28ed-48ce-8b8e-a82d754c6f9c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f677c761-8af0-4ab3-b20f-2436c20df40c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308159729Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308164149Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308168489Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308175759Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308227291Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308249341Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308255251Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 951f74f7-28ed-48ce-8b8e-a82d754c6f9c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f677c761-8af0-4ab3-b20f-2436c20df40c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308288892Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308293442Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308298522Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308301773Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308305593Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308310543Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308315653Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 951f74f7-28ed-48ce-8b8e-a82d754c6f9c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f677c761-8af0-4ab3-b20f-2436c20df40c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308349224Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308353444Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308356884Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308361374Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308364854Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 951f74f7-28ed-48ce-8b8e-a82d754c6f9c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f677c761-8af0-4ab3-b20f-2436c20df40c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308392255Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308396605Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308400255Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308403945Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:97"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308413136Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:99"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308420046Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 951f74f7-28ed-48ce-8b8e-a82d754c6f9c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f677c761-8af0-4ab3-b20f-2436c20df40c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308448826Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308454867Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308458317Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308461567Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:211"} {"msg":"No ledger in /pool/int/951f74f7-28ed-48ce-8b8e-a82d754c6f9c/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308511568Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f677c761-8af0-4ab3-b20f-2436c20df40c/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308525549Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308532939Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/951f74f7-28ed-48ce-8b8e-a82d754c6f9c/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308543569Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f677c761-8af0-4ab3-b20f-2436c20df40c/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308550559Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308558049Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.30857861Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:216"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.308592081Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:231"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.318941582Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:730"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.318974683Z","hostname":"buskin","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999946489s","next_cleanup":"Instant { tv_sec: 1083, tv_nsec: 967839976 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.319055505Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3842"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.319065795Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/951f74f7-28ed-48ce-8b8e-a82d754c6f9c/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.319121517Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f677c761-8af0-4ab3-b20f-2436c20df40c/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.319131257Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.322465278Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.32255557Z","hostname":"buskin","pid":1562,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:200::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n local_pref: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.322736715Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1074"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.322747225Z","hostname":"buskin","pid":1562,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.322792476Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/951f74f7-28ed-48ce-8b8e-a82d754c6f9c/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.322837648Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f677c761-8af0-4ab3-b20f-2436c20df40c/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.322845518Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.322850728Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.322875459Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/951f74f7-28ed-48ce-8b8e-a82d754c6f9c/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.322886239Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f677c761-8af0-4ab3-b20f-2436c20df40c/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.322895299Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.322899689Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1179"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.32290335Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:155"} {"msg":"Creating plan for the sled at fdb0:200::1","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.32290969Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:128"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.32292442Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/951f74f7-28ed-48ce-8b8e-a82d754c6f9c/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.322953911Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f677c761-8af0-4ab3-b20f-2436c20df40c/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.32330645Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage: Plan {\n rack_id: 890b6229-64f5-47ca-9c4f-4917b1f665e3,\n sleds: {\n [fdb0:200::1]:12346: StartSledAgentRequest {\n generation: 0,\n schema_version: 1,\n body: StartSledAgentRequestBody {\n id: da86b797-b25a-4724-8ccb-680ac8134295,\n rack_id: 890b6229-64f5-47ca-9c4f-4917b1f665e3,\n use_trust_quorum: false,\n is_lrtq_learner: false,\n subnet: Ipv6Subnet {\n net: Ipv6Net {\n addr: fd00:1122:3344:101::,\n width: 64,\n },\n },\n },\n },\n },\n config: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n local_pref: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.323519326Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:175"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.323538897Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1213"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.323569717Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/951f74f7-28ed-48ce-8b8e-a82d754c6f9c/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.323581758Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f677c761-8af0-4ab3-b20f-2436c20df40c/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.323727782Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.323882436Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.323926437Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:221","target_sled":"[fdb0:200::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: da86b797-b25a-4724-8ccb-680ac8134295, rack_id: 890b6229-64f5-47ca-9c4f-4917b1f665e3, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.32403307Z","hostname":"buskin","pid":1562,"remote_addr":"[fdb0:200::1]:63533","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: da86b797-b25a-4724-8ccb-680ac8134295, rack_id: 890b6229-64f5-47ca-9c4f-4917b1f665e3, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.324214875Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:359"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.324226615Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:374"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.324487942Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.324495513Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:644"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.324498803Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/server.rs:44"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.324505533Z","hostname":"buskin","pid":1562,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.324509273Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"No ledger in /pool/int/951f74f7-28ed-48ce-8b8e-a82d754c6f9c/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.324524014Z","hostname":"buskin","pid":1562,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f677c761-8af0-4ab3-b20f-2436c20df40c/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.324649167Z","hostname":"buskin","pid":1562,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.324655007Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:633"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.324658637Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:661"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.324662147Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:440"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.324707119Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.324711768Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:532","result":"DisksManagementResult { status: [] }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.326192119Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4139"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.32625157Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4521"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.358353343Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.380124285Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.397332642Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.408340332Z","hostname":"buskin","pid":1562,"sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:388"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.408368053Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.408379833Z","hostname":"buskin","pid":1562,"sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:400"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.408426454Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 93.418275ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.411323593Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"98e2e7c0-076b-4f18-9c6b-daf725309653\", \"content-length\": \"124\", \"date\": \"Sat, 07 Sep 2024 18:50:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"98e2e7c0-076b-4f18-9c6b-daf725309653\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_951f74f7-28ed-48ce-8b8e-a82d754c6f9c/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.421862979Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_951f74f7-28ed-48ce-8b8e-a82d754c6f9c/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.445528493Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_951f74f7-28ed-48ce-8b8e-a82d754c6f9c/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.445563873Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.450910139Z","hostname":"buskin","pid":1562,"sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:415"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.450922529Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:127"} {"msg":"Ensuring dataset oxi_951f74f7-28ed-48ce-8b8e-a82d754c6f9c/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.450926919Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:137"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.506220522Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 75.869637ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.506782828Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"080c8b10-8ff2-4a8a-bedf-4d9144bf3d26\", \"content-length\": \"124\", \"date\": \"Sat, 07 Sep 2024 18:50:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"080c8b10-8ff2-4a8a-bedf-4d9144bf3d26\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.537839532Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:172"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.584174281Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 110.108494ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.584793108Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"32fe777a-a94e-4cf6-8151-4c4e8066c261\", \"content-length\": \"124\", \"date\": \"Sat, 07 Sep 2024 18:50:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"32fe777a-a94e-4cf6-8151-4c4e8066c261\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.695623301Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 201.176489ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.696094314Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e0e77153-5ab2-4e3e-95c9-d3869bde565d\", \"content-length\": \"124\", \"date\": \"Sat, 07 Sep 2024 18:50:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e0e77153-5ab2-4e3e-95c9-d3869bde565d\" }"} {"msg":"Mounting oxi_951f74f7-28ed-48ce-8b8e-a82d754c6f9c/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.756500116Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:180"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.769286043Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:188"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.769361195Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:188"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.769385446Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:188"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.769401136Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:194"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.898731251Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 248.169236ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:33.899317538Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"933f0855-0987-40f6-81ba-360f3edb5f68\", \"content-length\": \"124\", \"date\": \"Sat, 07 Sep 2024 18:50:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"933f0855-0987-40f6-81ba-360f3edb5f68\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:34.148338846Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 231.894002ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:34.149146938Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6b797c21-f3a0-4843-ba50-c114a9bada3e\", \"content-length\": \"124\", \"date\": \"Sat, 07 Sep 2024 18:50:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6b797c21-f3a0-4843-ba50-c114a9bada3e\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:34.381632918Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 367.68456ms)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:34.382225804Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2bc258db-5d03-4b56-969d-081e685dcb47\", \"content-length\": \"124\", \"date\": \"Sat, 07 Sep 2024 18:50:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2bc258db-5d03-4b56-969d-081e685dcb47\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:34.505139365Z","hostname":"buskin","pid":1562,"sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:93"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:34.51049261Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:47084","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205","component":"producer-server","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:34.510516541Z","hostname":"buskin","pid":1562,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:73","component":"producer-server","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:34.510605853Z","hostname":"buskin","pid":1562,"file":"oximeter/producer/src/lib.rs:283","component":"producer-server","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:283","interval":"30s","address":"[fd00:1122:3344:101::1]:47084","producer_id":"da86b797-b25a-4724-8ccb-680ac8134295"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:34.511780145Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"273.274988ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:34.512431373Z","hostname":"buskin","pid":1562,"component":"VmmReservoirManager","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:34.512437503Z","hostname":"buskin","pid":1562,"component":"VmmReservoirManager","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:34.512501325Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:914","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:34.512507725Z","hostname":"buskin","pid":1562,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:73","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:34.750915126Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:34.787475429Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"402.433401ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:35.192022355Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"648.791176ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:35.842933328Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.9619126s"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:36.949193188Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2615"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:36.949233009Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2833"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:36.949239949Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2485"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:36.949245269Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2532"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:36.94924992Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2914"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:36.94925433Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:36.951179302Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2758"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:36.952452147Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2526"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:36.952546399Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:36.952675753Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 62.40124ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:37.271911559Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 87.913897ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:37.498425186Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 200.220641ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:37.674334128Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:37.80759153Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"4.75658395s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 471.537279ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:38.024763333Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.170551388s","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:38.59011258Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.155110096s","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:39.822589531Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.123504259s","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:41.19449451Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 966.24416ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:42.351564471Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:42.572383534Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"7.240899202s"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:43.625806777Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:205"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:43.625851468Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:429"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:43.625876849Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/951f74f7-28ed-48ce-8b8e-a82d754c6f9c/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:43.625895669Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f677c761-8af0-4ab3-b20f-2436c20df40c/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:43.626150256Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:43.626353892Z","hostname":"buskin","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:43.626369432Z","hostname":"buskin","pid":1562,"remote_addr":"[fdb0:200::1]:63533","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:43.626407943Z","hostname":"buskin","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.1.100, vlan_id: None, local_pref: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false, lldp: None }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:43.626421403Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3842"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:43.626474745Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:112","peer":"[fdb0:200::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:43.839025682Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:236","target_sled":"[fdb0:200::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:43.839210678Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/951f74f7-28ed-48ce-8b8e-a82d754c6f9c/config/rss-service-plan-v4.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:43.83930446Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f677c761-8af0-4ab3-b20f-2436c20df40c/config/rss-service-plan-v4.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:43.839339461Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:43.839377882Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:43.839397893Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:43.839413163Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:43.844171592Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4163","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:43.844192573Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4179"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:43.844198293Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:534"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.111708224Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:36747"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.112033013Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.112214287Z","hostname":"buskin","pid":1562,"uri":"/inventory","method":"GET","req_id":"0c0a0a87-ed7b-41bf-939a-1e430ceea3c3","remote_addr":"[fd00:1122:3344:101::1]:36747","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":341,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.143509798Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:38074"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.143624831Z","hostname":"buskin","pid":1562,"uri":"/sled-role","method":"GET","req_id":"881ec2bf-ea70-4a52-8183-0204b3ed0e5c","remote_addr":"[fd00:1122:3344:101::1]:38074","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":43,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.145138993Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/951f74f7-28ed-48ce-8b8e-a82d754c6f9c/config/.rss-service-plan-v4.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.145175663Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f677c761-8af0-4ab3-b20f-2436c20df40c/config/.rss-service-plan-v4.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.145507352Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.145818861Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:1001"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 5882cd91-c8a9-45db-b4ad-9a7289fe66a2, pool_id: 308c8624-6854-4c09-8fdb-403d3e7020c4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: ef8b359f-f47c-4ed0-9bd5-ab7a7010b669, pool_id: 7c49d716-0b82-4096-a83b-a43dd8cb57a5 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 91a10670-917c-4847-88af-6c08d48cec51, pool_id: faa0cc11-3e46-4294-868c-12831b857884 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 143fecea-ef54-4258-879f-96832bc6cbd5, pool_id: f5b9b071-b292-42ff-8f22-90770e6df42a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 614cc534-1c05-452f-826e-78d16b6aa53e, pool_id: 0b0f060f-15eb-447f-a739-7bce57c42c4b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 72da3f71-9fc0-4238-86ce-427fef2fda93, pool_id: 9d00b4ab-4f70-4f02-8fff-824dda813e40 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 34cad5af-ab67-4b75-8194-572995587f77, pool_id: 78ea79f6-adbf-4571-a535-37760fc84bd0 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: aec18675-7f24-4c08-8823-2f61d9b2a33a, pool_id: cb8c5357-055f-48a8-ad91-4c8525191c8d (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 40c54004-0c0b-446d-aca4-8c448ee9ceed, pool_id: 50801f60-af2f-4f08-8e46-106a71ec966e (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.178496879Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:375"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.178661104Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:53635"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.178872459Z","hostname":"buskin","pid":1562,"sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:856"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 5882cd91-c8a9-45db-b4ad-9a7289fe66a2, pool_id: 308c8624-6854-4c09-8fdb-403d3e7020c4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: ef8b359f-f47c-4ed0-9bd5-ab7a7010b669, pool_id: 7c49d716-0b82-4096-a83b-a43dd8cb57a5 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 91a10670-917c-4847-88af-6c08d48cec51, pool_id: faa0cc11-3e46-4294-868c-12831b857884 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 143fecea-ef54-4258-879f-96832bc6cbd5, pool_id: f5b9b071-b292-42ff-8f22-90770e6df42a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 614cc534-1c05-452f-826e-78d16b6aa53e, pool_id: 0b0f060f-15eb-447f-a739-7bce57c42c4b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 72da3f71-9fc0-4238-86ce-427fef2fda93, pool_id: 9d00b4ab-4f70-4f02-8fff-824dda813e40 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 34cad5af-ab67-4b75-8194-572995587f77, pool_id: 78ea79f6-adbf-4571-a535-37760fc84bd0 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: aec18675-7f24-4c08-8823-2f61d9b2a33a, pool_id: cb8c5357-055f-48a8-ad91-4c8525191c8d (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 40c54004-0c0b-446d-aca4-8c448ee9ceed, pool_id: 50801f60-af2f-4f08-8e46-106a71ec966e (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.178912011Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/951f74f7-28ed-48ce-8b8e-a82d754c6f9c/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.178920031Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f677c761-8af0-4ab3-b20f-2436c20df40c/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.178931461Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.178939271Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:866"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.178946372Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.178954232Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.178962842Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.178969452Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.317824276Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.317867808Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"308c8624-6854-4c09-8fdb-403d3e7020c4 (zpool)"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.570387212Z","hostname":"buskin","pid":1562,"component":"VmmReservoirManager","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/da86b797-b25a-4724-8ccb-680ac8134295): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-09-07T18:50:44.62806808Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.664558811Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 308c8624-6854-4c09-8fdb-403d3e7020c4 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.67222828Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.672317022Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.672329992Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.672343503Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.672361363Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.672470086Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.672482937Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.709970066Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_308c8624-6854-4c09-8fdb-403d3e7020c4/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.717528071Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"Looking for unencrypted datasets in oxp_308c8624-6854-4c09-8fdb-403d3e7020c4","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.893555386Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.901606225Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.901645046Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.901651566Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.901657286Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.901662066Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.911503794Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:44.911527905Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"7c49d716-0b82-4096-a83b-a43dd8cb57a5 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.004242744Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 7c49d716-0b82-4096-a83b-a43dd8cb57a5 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.011927893Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.011955903Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.011962143Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.011993564Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.012002435Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.012153279Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.012166089Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.051321143Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_7c49d716-0b82-4096-a83b-a43dd8cb57a5/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.060119532Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"Looking for unencrypted datasets in oxp_7c49d716-0b82-4096-a83b-a43dd8cb57a5","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.229715822Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.23735763Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.237391491Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.237399471Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.237405902Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.237423492Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.246497349Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.246533119Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"faa0cc11-3e46-4294-868c-12831b857884 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.337588925Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: faa0cc11-3e46-4294-868c-12831b857884 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.345414687Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.345478559Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.345488549Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.34550941Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.34551866Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.345673834Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.345682745Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.381979781Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_faa0cc11-3e46-4294-868c-12831b857884/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.389489645Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"Looking for unencrypted datasets in oxp_faa0cc11-3e46-4294-868c-12831b857884","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.564987526Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.572084199Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.57212569Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.57213648Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.57214393Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.572159771Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.581773482Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.581812563Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"f5b9b071-b292-42ff-8f22-90770e6df42a (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.67478479Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: f5b9b071-b292-42ff-8f22-90770e6df42a (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.682536561Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.682584162Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.682603443Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.682650624Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.682661774Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.682756127Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.682767167Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/da86b797-b25a-4724-8ccb-680ac8134295): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-09-07T18:50:45.686773416Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.719982569Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_f5b9b071-b292-42ff-8f22-90770e6df42a/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.728099829Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"Looking for unencrypted datasets in oxp_f5b9b071-b292-42ff-8f22-90770e6df42a","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.901860523Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.910381224Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.910431296Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.910440276Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.910446846Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.910452496Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.920310784Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:45.920341175Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"0b0f060f-15eb-447f-a739-7bce57c42c4b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.015767639Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 0b0f060f-15eb-447f-a739-7bce57c42c4b (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.024452555Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.024515757Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.024533957Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.024544057Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.024558798Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.024726202Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.024731982Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.061092041Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_0b0f060f-15eb-447f-a739-7bce57c42c4b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.069610632Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"Looking for unencrypted datasets in oxp_0b0f060f-15eb-447f-a739-7bce57c42c4b","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.247986921Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.256174103Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.256213455Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.256222815Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.256231175Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.256237145Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.265463036Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.265495067Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"9d00b4ab-4f70-4f02-8fff-824dda813e40 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.359104041Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: 9d00b4ab-4f70-4f02-8fff-824dda813e40 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.366863872Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.366888803Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.366926134Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.366945474Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.366952895Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.367066828Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.367073548Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.403088187Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_9d00b4ab-4f70-4f02-8fff-824dda813e40/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.411021212Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"Looking for unencrypted datasets in oxp_9d00b4ab-4f70-4f02-8fff-824dda813e40","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.589577736Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.597695537Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.597730678Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.597736888Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.597742508Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.597746308Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.60738752Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.607415101Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"78ea79f6-adbf-4571-a535-37760fc84bd0 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.700635045Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 78ea79f6-adbf-4571-a535-37760fc84bd0 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.708515519Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.70854947Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.70855588Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.708595881Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.708621502Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.708732995Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.708739645Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/da86b797-b25a-4724-8ccb-680ac8134295): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-09-07T18:50:46.713222057Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.745257498Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_78ea79f6-adbf-4571-a535-37760fc84bd0/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.753239105Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"Looking for unencrypted datasets in oxp_78ea79f6-adbf-4571-a535-37760fc84bd0","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.934546513Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.942302934Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.942324374Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.942359875Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.942366786Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.942370635Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.951335429Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:46.95136698Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"cb8c5357-055f-48a8-ad91-4c8525191c8d (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.043113313Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: cb8c5357-055f-48a8-ad91-4c8525191c8d (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.050151304Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.050180475Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.050185895Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.050201346Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.050208756Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.05034778Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.05035388Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.08606368Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_cb8c5357-055f-48a8-ad91-4c8525191c8d/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.094001896Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"Looking for unencrypted datasets in oxp_cb8c5357-055f-48a8-ad91-4c8525191c8d","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.26190093Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.26964183Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.269665631Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.269689922Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.269695942Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.269700002Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.280024913Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.280045483Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"50801f60-af2f-4f08-8e46-106a71ec966e (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.393153938Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:174","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 50801f60-af2f-4f08-8e46-106a71ec966e (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.400123277Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:218","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.400175239Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:220","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.400181469Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:224","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.400200389Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:226","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.40021168Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.400430146Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.400441056Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.441081941Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_50801f60-af2f-4f08-8e46-106a71ec966e/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.449547691Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:281"} {"msg":"Looking for unencrypted datasets in oxp_50801f60-af2f-4f08-8e46-106a71ec966e","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.625902794Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:367"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.633614844Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.633647435Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/951f74f7-28ed-48ce-8b8e-a82d754c6f9c/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.633712617Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.63384121Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 951f74f7-28ed-48ce-8b8e-a82d754c6f9c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f677c761-8af0-4ab3-b20f-2436c20df40c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 308c8624-6854-4c09-8fdb-403d3e7020c4 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 7c49d716-0b82-4096-a83b-a43dd8cb57a5 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: faa0cc11-3e46-4294-868c-12831b857884 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f5b9b071-b292-42ff-8f22-90770e6df42a (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 0b0f060f-15eb-447f-a739-7bce57c42c4b (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 9d00b4ab-4f70-4f02-8fff-824dda813e40 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 78ea79f6-adbf-4571-a535-37760fc84bd0 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: cb8c5357-055f-48a8-ad91-4c8525191c8d (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 50801f60-af2f-4f08-8e46-106a71ec966e (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.633894582Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.633900502Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/f677c761-8af0-4ab3-b20f-2436c20df40c/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.633964813Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.634157449Z","hostname":"buskin","pid":1562,"sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:860"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.634166099Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.634170359Z","hostname":"buskin","pid":1562,"sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:886","generation":"Generation(1)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.634174699Z","hostname":"buskin","pid":1562,"sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:891"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.634179829Z","hostname":"buskin","pid":1562,"sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:896"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.634183999Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:121","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.63418991Z","hostname":"buskin","pid":1562,"sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:902"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.634225981Z","hostname":"buskin","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:803","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.634232191Z","hostname":"buskin","pid":1562,"sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:907"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.634302243Z","hostname":"buskin","pid":1562,"uri":"/omicron-physical-disks","method":"PUT","req_id":"5204ccf4-7921-4814-b4a2-1a7311ae42ea","remote_addr":"[fd00:1122:3344:101::1]:53635","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":3455560,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 34fef005-d46a-42e7-b712-276d2ec90304, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 308c8624-6854-4c09-8fdb-403d3e7020c4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 308c8624-6854-4c09-8fdb-403d3e7020c4 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: de009927-0679-4b92-8b0c-f90a11fb5dc2, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 7c49d716-0b82-4096-a83b-a43dd8cb57a5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c49d716-0b82-4096-a83b-a43dd8cb57a5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: faa0cc11-3e46-4294-868c-12831b857884 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: faa0cc11-3e46-4294-868c-12831b857884 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.665903752Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.666047016Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:34714"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34fef005-d46a-42e7-b712-276d2ec90304, dataset_name: DatasetName { pool_name: ZpoolName { id: 308c8624-6854-4c09-8fdb-403d3e7020c4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.666264662Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.666270042Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 34fef005-d46a-42e7-b712-276d2ec90304, dataset_name: DatasetName { pool_name: ZpoolName { id: 308c8624-6854-4c09-8fdb-403d3e7020c4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/da86b797-b25a-4724-8ccb-680ac8134295): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-09-07T18:50:47.715178871Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de009927-0679-4b92-8b0c-f90a11fb5dc2, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c49d716-0b82-4096-a83b-a43dd8cb57a5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.719237262Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.719249462Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: de009927-0679-4b92-8b0c-f90a11fb5dc2, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c49d716-0b82-4096-a83b-a43dd8cb57a5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83, dataset_name: DatasetName { pool_name: ZpoolName { id: faa0cc11-3e46-4294-868c-12831b857884 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.771098941Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.771130032Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83, dataset_name: DatasetName { pool_name: ZpoolName { id: faa0cc11-3e46-4294-868c-12831b857884 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.820551765Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/951f74f7-28ed-48ce-8b8e-a82d754c6f9c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.820588136Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f677c761-8af0-4ab3-b20f-2436c20df40c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.820629687Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.820667668Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.864356956Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.864379177Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.864384247Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_34fef005-d46a-42e7-b712-276d2ec90304","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.91637706Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.920255086Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_de009927-0679-4b92-8b0c-f90a11fb5dc2","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.923793982Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_internal_dns_34fef005-d46a-42e7-b712-276d2ec90304","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.936345893Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.940724572Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_de009927-0679-4b92-8b0c-f90a11fb5dc2","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:47.944867444Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/da86b797-b25a-4724-8ccb-680ac8134295): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-09-07T18:50:48.717514645Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Profile for oxz_internal_dns_34fef005-d46a-42e7-b712-276d2ec90304:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:49.426160907Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:49.426242229Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_de009927-0679-4b92-8b0c-f90a11fb5dc2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:49.904777157Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_internal_dns_34fef005-d46a-42e7-b712-276d2ec90304 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:49.904842458Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_34fef005-d46a-42e7-b712-276d2ec90304","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:50.388316999Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:50.388389361Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:50.388400792Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"Booting oxz_internal_dns_de009927-0679-4b92-8b0c-f90a11fb5dc2 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:50.388514815Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_de009927-0679-4b92-8b0c-f90a11fb5dc2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_internal_dns_34fef005-d46a-42e7-b712-276d2ec90304\") failed: Property not found. retry in 42.994328ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:50.402584567Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_34fef005-d46a-42e7-b712-276d2ec90304","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_internal_dns_af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:50.402641829Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_internal_dns_34fef005-d46a-42e7-b712-276d2ec90304\") failed: Property not found. retry in 76.262247ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:50.454662413Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_34fef005-d46a-42e7-b712-276d2ec90304","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_de009927-0679-4b92-8b0c-f90a11fb5dc2\") failed: Property not found. retry in 33.524965ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:50.522368853Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_de009927-0679-4b92-8b0c-f90a11fb5dc2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83\") failed: Property not found. retry in 29.803651ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:50.531914553Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_34fef005-d46a-42e7-b712-276d2ec90304\") failed: Property not found. retry in 245.000682ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:50.538491121Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_34fef005-d46a-42e7-b712-276d2ec90304","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_de009927-0679-4b92-8b0c-f90a11fb5dc2\") failed: Property not found. retry in 125.882768ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:50.564060907Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_de009927-0679-4b92-8b0c-f90a11fb5dc2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83\") failed: Property not found. retry in 118.798998ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:50.570599134Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83\") failed: Property not found. retry in 294.90481ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:50.720804127Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_de009927-0679-4b92-8b0c-f90a11fb5dc2\") failed: Property not found. retry in 150.85861ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:50.732508355Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_de009927-0679-4b92-8b0c-f90a11fb5dc2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_34fef005-d46a-42e7-b712-276d2ec90304\") failed: Property not found. retry in 350.175412ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:50.797881272Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_34fef005-d46a-42e7-b712-276d2ec90304","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_de009927-0679-4b92-8b0c-f90a11fb5dc2\") failed: Property not found. retry in 482.842063ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:50.898034345Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_de009927-0679-4b92-8b0c-f90a11fb5dc2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83\") failed: Property not found. retry in 499.253504ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:51.030013182Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_34fef005-d46a-42e7-b712-276d2ec90304\") failed: Property not found. retry in 1.199606165s","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:51.159025519Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_34fef005-d46a-42e7-b712-276d2ec90304","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_de009927-0679-4b92-8b0c-f90a11fb5dc2\") failed: Property not found. retry in 855.017558ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:51.390420609Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_de009927-0679-4b92-8b0c-f90a11fb5dc2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83\") failed: Property not found. retry in 487.972059ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:51.539822019Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83\") failed: Property not found. retry in 1.084183848s","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:52.038257737Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_de009927-0679-4b92-8b0c-f90a11fb5dc2\") failed: Property not found. retry in 1.379917989s","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:52.256581792Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_de009927-0679-4b92-8b0c-f90a11fb5dc2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_34fef005-d46a-42e7-b712-276d2ec90304\") failed: Property not found. retry in 1.083369259s","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:52.368194555Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_34fef005-d46a-42e7-b712-276d2ec90304","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83\") failed: Property not found. retry in 1.25866393s","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:53.131681498Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:53.324416366Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4187"} {"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:53.324458447Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4197","gateway":"fd00:1122:3344:101::1"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:53.330339217Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4276"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:53.449208009Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4324"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:53.44924313Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4351"} {"msg":"wait for service Some(\"oxz_internal_dns_34fef005-d46a-42e7-b712-276d2ec90304\") failed: Property not found. retry in 720.920645ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:53.461674457Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_34fef005-d46a-42e7-b712-276d2ec90304","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:53.488411774Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4364"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:53.488444005Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4228"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:53.542147175Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4270"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:53.542176965Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4331","rack_subnet":"fd00:1122:3344::"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:53.561992624Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4342"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:53.562027675Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4384"} {"msg":"wait for service Some(\"oxz_internal_dns_de009927-0679-4b92-8b0c-f90a11fb5dc2\") failed: Property not found. retry in 1.16203368s","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:53.645032001Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_de009927-0679-4b92-8b0c-f90a11fb5dc2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:53.649916634Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4416"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:53.649924494Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4422"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:53.757815697Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4463"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:53.757856018Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:366"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/da86b797-b25a-4724-8ccb-680ac8134295): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:53.791127752Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"} {"msg":"Failed to get switch ID from MGS (retrying in 41.21435ms)","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:53.791185854Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:385","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Failed to get switch ID from MGS (retrying in 144.584285ms)","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:53.83335177Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:385","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Failed to get switch ID from MGS (retrying in 143.57596ms)","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:53.979517742Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:385","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.125679635Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:414"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.158905568Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:676"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.159018711Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:714"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.159438003Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:717","uptime":"0"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.159465564Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:436","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.196629254Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::2]:42217"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.196880571Z","hostname":"buskin","pid":1562,"uri":"/sled-identifiers","method":"GET","req_id":"f2cecdb6-603c-4361-8680-de122182769c","remote_addr":"[fd00:1122:3344:101::2]:42217","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":182,"response_code":"200"} {"msg":"wait for service Some(\"oxz_internal_dns_34fef005-d46a-42e7-b712-276d2ec90304\") failed: Property not found. retry in 913.088866ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:54.201674711Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_34fef005-d46a-42e7-b712-276d2ec90304","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.20312861Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4014"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.255597537Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.255663149Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.255672209Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.255679469Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.255686659Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.255692419Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.25569827Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.2557052Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.25571125Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.25571673Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.25572393Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.25573113Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.255737431Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.255747391Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.255752161Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.255756821Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.255761211Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.255765691Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.255770152Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.255774432Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.255778932Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.255783352Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:54.255787732Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"wait for service Some(\"oxz_internal_dns_af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83\") failed: Property not found. retry in 1.173496415s","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:54.40029494Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_de009927-0679-4b92-8b0c-f90a11fb5dc2\") failed: Property not found. retry in 1.127046062s","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:54.816031341Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_de009927-0679-4b92-8b0c-f90a11fb5dc2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_34fef005-d46a-42e7-b712-276d2ec90304\") failed: Property not found. retry in 630.661055ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:55.124027472Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_34fef005-d46a-42e7-b712-276d2ec90304","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:55.964200918Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_internal_dns_de009927-0679-4b92-8b0c-f90a11fb5dc2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:55.964247279Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_internal_dns_34fef005-d46a-42e7-b712-276d2ec90304"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:55.96425366Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_internal_dns_af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83"} {"msg":"Writing ledger to /pool/int/951f74f7-28ed-48ce-8b8e-a82d754c6f9c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:55.96426261Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f677c761-8af0-4ab3-b20f-2436c20df40c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:55.964595989Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:55.964933708Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"7a25517f-6838-43fb-a6bd-b7c0cf6fb956","remote_addr":"[fd00:1122:3344:101::1]:34714","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8298823,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:55.966798799Z","hostname":"buskin","pid":1562,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.01466652Z","hostname":"buskin","pid":1562,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.070365744Z","hostname":"buskin","pid":1562,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.115992014Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:611"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.116002904Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.150527313Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.15153513Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.151547081Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 34fef005-d46a-42e7-b712-276d2ec90304, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 308c8624-6854-4c09-8fdb-403d3e7020c4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 308c8624-6854-4c09-8fdb-403d3e7020c4 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: de009927-0679-4b92-8b0c-f90a11fb5dc2, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 7c49d716-0b82-4096-a83b-a43dd8cb57a5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c49d716-0b82-4096-a83b-a43dd8cb57a5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: faa0cc11-3e46-4294-868c-12831b857884 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: faa0cc11-3e46-4294-868c-12831b857884 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 434193fa-4a13-4c27-8229-3a8b4e208861, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 308c8624-6854-4c09-8fdb-403d3e7020c4 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6ba9518e-b713-4094-ae2c-ac769b602c51, kind: Service { id: 434193fa-4a13-4c27-8229-3a8b4e208861 }, name: Name(\"ntp-434193fa-4a13-4c27-8229-3a8b4e208861\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 72])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.180999161Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.181154586Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:38331"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 34fef005-d46a-42e7-b712-276d2ec90304, dataset_name: DatasetName { pool_name: ZpoolName { id: 308c8624-6854-4c09-8fdb-403d3e7020c4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.181545906Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.181554666Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: 34fef005-d46a-42e7-b712-276d2ec90304, dataset_name: DatasetName { pool_name: ZpoolName { id: 308c8624-6854-4c09-8fdb-403d3e7020c4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de009927-0679-4b92-8b0c-f90a11fb5dc2, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c49d716-0b82-4096-a83b-a43dd8cb57a5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.200998265Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.201068557Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: de009927-0679-4b92-8b0c-f90a11fb5dc2, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c49d716-0b82-4096-a83b-a43dd8cb57a5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83, dataset_name: DatasetName { pool_name: ZpoolName { id: faa0cc11-3e46-4294-868c-12831b857884 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.216751963Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.216797034Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:1065","request":"NewFilesystemRequest { dataset_id: af9e1e4c-5a5a-45bb-9c51-f2806bc3ae83, dataset_name: DatasetName { pool_name: ZpoolName { id: faa0cc11-3e46-4294-868c-12831b857884 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.231857794Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/951f74f7-28ed-48ce-8b8e-a82d754c6f9c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.231909785Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f677c761-8af0-4ab3-b20f-2436c20df40c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.23209158Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.232177322Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.242566435Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.275897161Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.277964757Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.277977157Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.30862176Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:424","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 208, 72]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.308650161Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1234","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.309077673Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.309082033Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.309312139Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.31044421Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.334018581Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.351454335Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.97560973Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861 zone","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:50:56.975743414Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861\") failed: Property not found. retry in 25.110846ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:57.109440048Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861\") failed: Property not found. retry in 110.626158ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:57.142340442Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861\") failed: Property not found. retry in 195.262244ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:57.282663136Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861\") failed: Property not found. retry in 368.131268ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:57.48623176Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861\") failed: Property not found. retry in 691.002695ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:57.863866795Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861\") failed: Property not found. retry in 695.334302ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:58.56493524Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861\") failed: Property not found. retry in 787.478141ms","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:50:59.270229812Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861\") failed: Property not found. retry in 1.125612122s","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:00.068320764Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/da86b797-b25a-4724-8ccb-680ac8134295): error trying to connect: dns error: no record found for Query { name: Name(\"_nexus._tcp.control-plane.oxide.internal.\"), query_type: SRV, query_class: IN }","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:00.795033748Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"sled-agent/src/nexus.rs:571"} {"msg":"wait for service Some(\"oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861\") failed: Property not found. retry in 1.231081086s","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:01.204041185Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:02.45545879Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3226","zone":"oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861"} {"msg":"Writing ledger to /pool/int/951f74f7-28ed-48ce-8b8e-a82d754c6f9c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:02.455517212Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f677c761-8af0-4ab3-b20f-2436c20df40c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:02.455862741Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:02.456262302Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"41ab6306-6d07-4322-918d-68001a649050","remote_addr":"[fd00:1122:3344:101::1]:38331","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":6274940,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:02.458683658Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:02.488833967Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:02.488994382Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:41731"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-09-07T18:51:02.498036568Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3828"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:02.498157441Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"993415f6-b60e-43ce-af7e-8ca47b5bee17","remote_addr":"[fd00:1122:3344:101::1]:41731","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9014,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:02.500763742Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:02.663408633Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:02.663541996Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:56130"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-09-07T18:51:06.671414566Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3828"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:06.671608921Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"3dfbd346-a39c-4b31-9eba-b26fe0c0d678","remote_addr":"[fd00:1122:3344:101::1]:56130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4007945,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:06.674718885Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:06.851622924Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:06.851767378Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:48677"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-09-07T18:51:06.86032805Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3828"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:06.860434373Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"6a164255-b277-48b8-bef4-e231fe44d6d2","remote_addr":"[fd00:1122:3344:101::1]:48677","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8511,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:06.861954235Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:07.093768955Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:07.093928559Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:48940"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_434193fa-4a13-4c27-8229-3a8b4e208861': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-09-07T18:51:07.102745649Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3828"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:07.102891543Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"a9474b3f-ac68-481c-919e-a9bd7e6e0958","remote_addr":"[fd00:1122:3344:101::1]:48940","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8828,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:07.104411244Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:07.317007593Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:07.317139677Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:37584"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:11.326125616Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"9fb6e501-77b2-49a2-b356-bf3b235784a2","remote_addr":"[fd00:1122:3344:101::1]:37584","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":4008844,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:11.329376154Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:11.329386695Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:11.645195539Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:11.645330733Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:36753"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:11.654637306Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"b54e7979-419d-4ec3-bde9-bc1979b52dfe","remote_addr":"[fd00:1122:3344:101::1]:36753","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":9104,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:11.657316989Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:11.657327049Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:11.965470774Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:11.965566267Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:40954"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:11.973686938Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"d67861d4-239d-4662-9e65-74e1ebe1bf7c","remote_addr":"[fd00:1122:3344:101::1]:40954","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8044,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:11.975306612Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:11.975313932Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:12.30340028Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:12.303528784Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:49992"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:12.31150062Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"b9997b2e-941b-4402-acf8-9e1650165ae4","remote_addr":"[fd00:1122:3344:101::1]:49992","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7877,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:12.313183056Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:12.313189026Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:12.856775352Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:12.856943057Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:40707"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:12.864597905Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"dfdd2bc3-ac5a-4ce9-8d54-5323c0b195b0","remote_addr":"[fd00:1122:3344:101::1]:40707","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7604,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:12.867469413Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:12.867480443Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:13.274639759Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:13.274762703Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:40398"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:13.282100532Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"8b55de6d-5b12-4ecb-a3cd-fae0848579a6","remote_addr":"[fd00:1122:3344:101::1]:40398","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":7218,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:13.283944052Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:13.283952332Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:14.079954989Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:14.080076542Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:50629"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:14.08847359Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"1a69fc9d-52c3-45da-bbbd-3b10cb0095af","remote_addr":"[fd00:1122:3344:101::1]:50629","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8273,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:14.090023002Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:14.090030702Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:14.448537478Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:14.448667761Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:59832"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:14.457433009Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"6a677db1-32de-4a89-83c1-e29cffadfd88","remote_addr":"[fd00:1122:3344:101::1]:59832","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8655,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:14.459251249Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:14.459258649Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:14.855696535Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"368.11413ms"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:15.286514865Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:15.28669328Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:42585"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:15.295322995Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"eff7152e-7828-4a3a-9811-fc8b86a809e4","remote_addr":"[fd00:1122:3344:101::1]:42585","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8578,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:15.297401831Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:15.297409041Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:16.034904527Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:16.035094062Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:36281"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:16.044016015Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"fc01ad9f-da2e-40bd-993e-a2985361dd7b","remote_addr":"[fd00:1122:3344:101::1]:36281","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8881,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:16.04567036Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:16.04567587Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:16.836454695Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:16.83662639Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:55862"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:16.844992427Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"438769e5-7dcc-46bf-8a79-8c6cbffea393","remote_addr":"[fd00:1122:3344:101::1]:55862","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8319,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:16.848450241Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:16.848480592Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:17.771759237Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:17.771921962Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:39173"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:17.780522486Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"3cab679e-03c5-436d-ad6c-04d13545a28d","remote_addr":"[fd00:1122:3344:101::1]:39173","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8556,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:17.783135997Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:17.783146427Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:19.495643435Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:19.49583119Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:47071"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:19.504124955Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"a0667ce2-56c3-4b15-ade9-e99ccd615c20","remote_addr":"[fd00:1122:3344:101::1]:47071","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8228,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:19.506442278Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:19.506450679Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:21.570155592Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:21.570350967Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:55442"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:21.579889607Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"156e5aaa-3dc7-4abc-8ef4-0fa74e0136c3","remote_addr":"[fd00:1122:3344:101::1]:55442","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":9473,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:21.58221123Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:21.58221727Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:23.533457098Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:200::2]:40063"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:23.533572021Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"f9b883d1-86fc-41f0-a23d-4f2ac8b3204c","remote_addr":"[fdb0:200::2]:40063","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":54,"response_code":"200"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:24.032536103Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:24.032691237Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:35211"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:24.041599039Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"ed142a58-9da7-4e54-a05f-6f968955d083","remote_addr":"[fd00:1122:3344:101::1]:35211","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8859,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:24.045629168Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:24.045639519Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:26.568635737Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:26.568857413Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:63802"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:26.577672892Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"3bb65bca-fb67-4923-aec8-cf32e669dd40","remote_addr":"[fd00:1122:3344:101::1]:63802","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8754,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:26.579583944Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:26.579596425Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:28.73754156Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:28.737718625Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:43174"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:28.74672417Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"d0fb87e7-0b05-43c9-b644-3c91758a0126","remote_addr":"[fd00:1122:3344:101::1]:43174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8957,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:28.750872122Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:28.750882453Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:30.259440087Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"414.47922ms"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:33.186324754Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:33.186534849Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:51873"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:33.195995307Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"1d61e761-5f3c-496d-9a94-f0c9c58c9df4","remote_addr":"[fd00:1122:3344:101::1]:51873","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":9391,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:33.198651489Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:33.198658669Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:38.71424683Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:38.714440815Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:41651"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:38.723572004Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"d8bc4e09-9f17-4997-a220-d20a53d7e5df","remote_addr":"[fd00:1122:3344:101::1]:41651","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":9095,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:38.726671958Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:38.726678988Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:44.632228797Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:44.632357101Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:64226"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:44.640995346Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"3331ac4a-bedc-4957-ac7d-40b75c196816","remote_addr":"[fd00:1122:3344:101::1]:64226","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8473,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:44.643489044Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:44.643520295Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:45.708841572Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"1.143934111s"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:51.792426781Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:51.792613386Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:46570"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:51.801531388Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"164e9474-92c3-4f29-806b-4b23b82849df","remote_addr":"[fd00:1122:3344:101::1]:46570","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8867,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:51.803998166Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:51:51.804005376Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:53.567084409Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:200::2]:34453"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:51:53.567181071Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"8a57604b-2e60-4e6d-9bf2-cbe616931451","remote_addr":"[fdb0:200::2]:34453","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":51,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:52:01.88760855Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"1.468292428s"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:52:02.205415258Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:52:02.205541391Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:57239"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:52:02.214662669Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"ecaf3efc-1f96-447c-b6c3-5bad37531f0f","remote_addr":"[fd00:1122:3344:101::1]:57239","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8963,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:52:02.21727066Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:52:02.21727803Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:52:11.448486706Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:52:11.448693962Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:37509"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:52:11.457943443Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"189922e6-b4ba-487f-b746-4b3cd7f05148","remote_addr":"[fd00:1122:3344:101::1]:37509","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":9204,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:52:11.460523133Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:52:11.460530183Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:52:18.392701439Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"2.323840656s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:52:23.59926534Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:200::2]:40802"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:52:23.599353272Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"fbc7ac5d-04e2-43e6-b8f4-4fb96e6202be","remote_addr":"[fdb0:200::2]:40802","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":51,"response_code":"200"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:52:25.332918023Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:52:25.333057946Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:44603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:52:25.34274465Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"67d42b66-cfe3-476e-a0a6-f0ca22df29b2","remote_addr":"[fd00:1122:3344:101::1]:44603","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":9559,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:52:25.344943629Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:52:25.34494889Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:52:32.963785229Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:52:32.963958474Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:42988"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:52:32.972812174Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"8f492c2c-d612-4efb-bc4a-3ed43acfb3de","remote_addr":"[fd00:1122:3344:101::1]:42988","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8797,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:52:32.977045769Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:52:32.977052709Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:52:35.75137299Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"7.025775571s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:52:53.634064415Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:200::2]:40072"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:52:53.634110647Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"842483e2-4d47-4c6e-9ce3-9a3938764d74","remote_addr":"[fdb0:200::2]:40072","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":37,"response_code":"200"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:52:54.824160874Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:52:54.824382889Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:51895"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:52:54.834453923Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"6c2aeddf-2cd7-46e7-9d6f-1bf7625d8c5c","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":10005,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:52:54.838090342Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:52:54.838105102Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:52:57.812794988Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"9.936310574s"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:53:12.5189965Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:53:12.519217346Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:61264"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:53:12.529491815Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"c54bdacb-1f9d-45f8-9800-492fadb23717","remote_addr":"[fd00:1122:3344:101::1]:61264","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":10149,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:53:12.53190152Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:53:12.5319104Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:53:22.783940415Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"26.29689891s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:53:23.672457647Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:200::2]:50692"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:53:23.673464174Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"af3c3a6c-d1dd-4e25-86f0-53cecede0ade","remote_addr":"[fdb0:200::2]:50692","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":959,"response_code":"200"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:53:39.368419654Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:53:39.368543077Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:65362"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:53:39.37784221Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"83e27053-10eb-425e-9efd-9043017c943f","remote_addr":"[fd00:1122:3344:101::1]:65362","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":9126,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:53:39.380330277Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:53:39.380336298Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:53:53.705711352Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:200::2]:43459"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:53:53.706572165Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"4f76eb7d-9f70-4e8e-b3aa-97994ee75203","remote_addr":"[fdb0:200::2]:43459","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":836,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:54:04.115440681Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"60.174172579s"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:54:05.951020885Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:54:05.951155268Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:50030"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:54:05.959776683Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"1d162b1b-abc4-426a-bedf-e1d90b4a9322","remote_addr":"[fd00:1122:3344:101::1]:50030","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":8441,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:54:05.96223652Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:54:05.96224559Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:54:23.743052106Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:200::2]:35119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:54:23.743880578Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"5111a5ca-689a-4f76-a8f5-7325b34f6f4a","remote_addr":"[fdb0:200::2]:35119","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":798,"response_code":"200"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:54:48.05076044Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:54:48.050959995Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:59643"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:54:48.061343948Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"6b81a298-98ce-4f9a-b5a6-c63391ce011b","remote_addr":"[fd00:1122:3344:101::1]:59643","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":10316,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:54:48.063185098Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:54:48.063194168Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:54:53.775946629Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:200::2]:33174"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:54:53.776050762Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"f308cfd3-2fd3-4a4b-b43b-179bfa2b6f53","remote_addr":"[fdb0:200::2]:33174","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":62,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:55:19.326751663Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"da86b797-b25a-4724-8ccb-680ac8134295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"181.438348896s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:55:23.808489463Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:200::2]:50966"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:55:23.808568415Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"a4349bfc-99f9-4fc5-9408-e07a656ba494","remote_addr":"[fdb0:200::2]:50966","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":38,"response_code":"200"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:55:35.918361416Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:55:35.91850345Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:34862"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:55:35.928309126Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"c13dd88e-57fd-4bfc-a40a-c09b78b93b69","remote_addr":"[fd00:1122:3344:101::1]:34862","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":9610,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:55:35.930717402Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-09-07T18:55:35.930723442Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:55:47.634983087Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:55:53.840834031Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:775","remote_addr":"[fdb0:200::2]:43444"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-09-07T18:55:53.840920033Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"adc68a25-520c-48a5-a070-be60a694b143","remote_addr":"[fdb0:200::2]:43444","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/06c8dab/dropshot/src/server.rs:914","latency_us":48,"response_code":"200"}