[ Oct 16 04:19:01 Enabled. ] [ Oct 16 04:19:01 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Oct 16 04:19:01 Method "start" exited with status 0. ] [ Oct 16 04:19:01 Rereading configuration. ] [ Oct 16 04:19:01 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:01.104982681Z","hostname":"buskin","pid":1572,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:01.105152463Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:01.158430613Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:01.158477944Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:01.271307362Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:01.280490688Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:01.288537379Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:01.891134425Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:200::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.031114434Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:126"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.031138755Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:138"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.031200835Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:151"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.031266546Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.031294916Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.031305117Z","hostname":"buskin","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:717"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.034333995Z","hostname":"buskin","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:624"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:02.034340605Z","hostname":"buskin","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:766"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.034350345Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:187"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.034359835Z","hostname":"buskin","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:825"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.034376065Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.034433656Z","hostname":"buskin","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.034502277Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.034522687Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.034530797Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.045705008Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.045727688Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"af85c7e6-e4e0-4385-860c-e4f6171b504b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.143775761Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: af85c7e6-e4e0-4385-860c-e4f6171b504b (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.554283711Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.554327142Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.554332612Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.554345002Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.554355403Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.554373653Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.554378383Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.56293465Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.562957481Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"76b73db8-8c3d-46ae-bed9-6b12bffa8365 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:02.652182052Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 76b73db8-8c3d-46ae-bed9-6b12bffa8365 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063065527Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063110778Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: af85c7e6-e4e0-4385-860c-e4f6171b504b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 76b73db8-8c3d-46ae-bed9-6b12bffa8365 (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063147518Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063154328Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063165268Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063173078Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063178248Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063183348Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: af85c7e6-e4e0-4385-860c-e4f6171b504b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 76b73db8-8c3d-46ae-bed9-6b12bffa8365 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063199979Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063204839Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063209649Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063228809Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063232909Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063237489Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063242189Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: af85c7e6-e4e0-4385-860c-e4f6171b504b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 76b73db8-8c3d-46ae-bed9-6b12bffa8365 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063258909Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.06326504Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063269329Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.06327407Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.0632785Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: af85c7e6-e4e0-4385-860c-e4f6171b504b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 76b73db8-8c3d-46ae-bed9-6b12bffa8365 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.06329668Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.06331023Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.06331479Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.06331902Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.06332404Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.06332911Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.06333524Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: af85c7e6-e4e0-4385-860c-e4f6171b504b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 76b73db8-8c3d-46ae-bed9-6b12bffa8365 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063356821Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063361871Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063367211Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063370841Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063374931Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: af85c7e6-e4e0-4385-860c-e4f6171b504b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 76b73db8-8c3d-46ae-bed9-6b12bffa8365 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063403231Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063408001Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063412771Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063420631Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063424922Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063430091Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: af85c7e6-e4e0-4385-860c-e4f6171b504b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 76b73db8-8c3d-46ae-bed9-6b12bffa8365 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063461062Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063465622Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063471412Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063475022Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063480062Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063497412Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063501532Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063508912Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: af85c7e6-e4e0-4385-860c-e4f6171b504b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 76b73db8-8c3d-46ae-bed9-6b12bffa8365 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063544113Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063548973Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063552653Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063556643Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063559903Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063563763Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: af85c7e6-e4e0-4385-860c-e4f6171b504b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 76b73db8-8c3d-46ae-bed9-6b12bffa8365 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063591974Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:97"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063598864Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063603524Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063613364Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:99"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063618404Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063621804Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: af85c7e6-e4e0-4385-860c-e4f6171b504b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 76b73db8-8c3d-46ae-bed9-6b12bffa8365 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063654534Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063659085Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063662434Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:211"} {"msg":"No ledger in /pool/int/af85c7e6-e4e0-4385-860c-e4f6171b504b/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063710965Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/76b73db8-8c3d-46ae-bed9-6b12bffa8365/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063724775Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063732275Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/af85c7e6-e4e0-4385-860c-e4f6171b504b/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063747086Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/76b73db8-8c3d-46ae-bed9-6b12bffa8365/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063751866Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063755616Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063782196Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:216"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.063786666Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:231"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.074606792Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:739"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.074668973Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3924"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.074677463Z","hostname":"buskin","pid":1572,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999920269s","next_cleanup":"Instant { tv_sec: 1100, tv_nsec: 681504382 }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.074716973Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/af85c7e6-e4e0-4385-860c-e4f6171b504b/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.074768054Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/76b73db8-8c3d-46ae-bed9-6b12bffa8365/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.077756662Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.078181937Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.080161012Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:51","local_addr":"[fdb0:200::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.080406195Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1073"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.080411975Z","hostname":"buskin","pid":1572,"component":"DnsResolver","component":"RSS","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.080453696Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/af85c7e6-e4e0-4385-860c-e4f6171b504b/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.080493226Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/76b73db8-8c3d-46ae-bed9-6b12bffa8365/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.080502826Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.080507566Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1136"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.080532937Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/af85c7e6-e4e0-4385-860c-e4f6171b504b/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.080554617Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/76b73db8-8c3d-46ae-bed9-6b12bffa8365/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.080562407Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.080566257Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1178"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.080569687Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:156"} {"msg":"Creating plan for the sled at fdb0:200::1","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.080573127Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:129"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.080589817Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/af85c7e6-e4e0-4385-860c-e4f6171b504b/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.080633308Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/76b73db8-8c3d-46ae-bed9-6b12bffa8365/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.080914931Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage: Plan {\n rack_id: 4e23417a-88af-4f6c-9590-90adf9bd40c0,\n sleds: {\n [fdb0:200::1]:12346: StartSledAgentRequest {\n generation: 0,\n schema_version: 1,\n body: StartSledAgentRequestBody {\n id: 832ac8b0-9619-4005-8cc6-03345e059ddb (sled),\n rack_id: 4e23417a-88af-4f6c-9590-90adf9bd40c0,\n use_trust_quorum: false,\n is_lrtq_learner: false,\n subnet: Ipv6Subnet {\n net: Ipv6Net {\n addr: fd00:1122:3344:101::,\n width: 64,\n },\n },\n },\n },\n },\n config: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.081173955Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:176"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.081179525Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1212"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.081218915Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/af85c7e6-e4e0-4385-860c-e4f6171b504b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.081226905Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/76b73db8-8c3d-46ae-bed9-6b12bffa8365/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.081372437Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.081502109Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.081554979Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:200::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 832ac8b0-9619-4005-8cc6-03345e059ddb (sled), rack_id: 4e23417a-88af-4f6c-9590-90adf9bd40c0, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.082025895Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4221"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.082099016Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4603"} {"msg":"Loaded keys and certs","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.087047959Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:200::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/client.rs:50"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.092873882Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:200::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.093113165Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:200::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.097896685Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.097961496Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.098092367Z","hostname":"buskin","pid":1572,"remote_addr":"[fdb0:200::1]:45629","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:74"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 832ac8b0-9619-4005-8cc6-03345e059ddb (sled), rack_id: 4e23417a-88af-4f6c-9590-90adf9bd40c0, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.098235749Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:362"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.098246709Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:377"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.09828815Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/server.rs:44"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.09830049Z","hostname":"buskin","pid":1572,"component":"DnsResolver","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.09830767Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.0983127Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:644"} {"msg":"No ledger in /pool/int/af85c7e6-e4e0-4385-860c-e4f6171b504b/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.09832596Z","hostname":"buskin","pid":1572,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/76b73db8-8c3d-46ae-bed9-6b12bffa8365/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.098340921Z","hostname":"buskin","pid":1572,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.098354961Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:633"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.098378441Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:661"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.098382771Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:440"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.098389331Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.098397771Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.098402051Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:532","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.115857931Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.137163338Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.154641308Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.163989316Z","hostname":"buskin","pid":1572,"sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:391"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.164032946Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.164040776Z","hostname":"buskin","pid":1572,"sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:403"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.164094477Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 66.348708ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.167193816Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a4b79164-0d5c-44f3-8bff-57e0cf6f52f1\", \"content-length\": \"124\", \"date\": \"Wed, 16 Oct 2024 04:19:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a4b79164-0d5c-44f3-8bff-57e0cf6f52f1\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_af85c7e6-e4e0-4385-860c-e4f6171b504b/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.176902568Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_af85c7e6-e4e0-4385-860c-e4f6171b504b/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.198886685Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_af85c7e6-e4e0-4385-860c-e4f6171b504b/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.198925895Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.203908268Z","hostname":"buskin","pid":1572,"sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:418"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.203915628Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:128"} {"msg":"Ensuring dataset oxi_af85c7e6-e4e0-4385-860c-e4f6171b504b/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.203922498Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:138"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.234786426Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 158.418409ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.235318372Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2c83d396-fef8-451c-bcb4-79f5817a0c9e\", \"content-length\": \"124\", \"date\": \"Wed, 16 Oct 2024 04:19:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2c83d396-fef8-451c-bcb4-79f5817a0c9e\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.295318227Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:173"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.395457696Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 72.303348ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.396045243Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"385836d4-548f-4c75-8af7-c7bdfdfe8ef0\", \"content-length\": \"124\", \"date\": \"Wed, 16 Oct 2024 04:19:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"385836d4-548f-4c75-8af7-c7bdfdfe8ef0\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.469420716Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 256.645105ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.46974385Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d6bcbe6a-4350-42bf-8838-2cba274bea5f\", \"content-length\": \"124\", \"date\": \"Wed, 16 Oct 2024 04:19:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d6bcbe6a-4350-42bf-8838-2cba274bea5f\" }"} {"msg":"Mounting oxi_af85c7e6-e4e0-4385-860c-e4f6171b504b/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.513993546Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:181"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.526593605Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.526671695Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.526688966Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.526703506Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:195"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.7266703Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 165.64419ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.73146093Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"15a98926-9e82-4435-92c4-eb8782cd92d2\", \"content-length\": \"124\", \"date\": \"Wed, 16 Oct 2024 04:19:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"15a98926-9e82-4435-92c4-eb8782cd92d2\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.897960513Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 150.64847ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:03.898600991Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"60243f16-acb7-4cb2-8350-5930af200246\", \"content-length\": \"124\", \"date\": \"Wed, 16 Oct 2024 04:19:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"60243f16-acb7-4cb2-8350-5930af200246\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:04.049856242Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 185.434194ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:04.050283507Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a4e050b2-edfc-4534-8f04-6376a3fef961\", \"content-length\": \"124\", \"date\": \"Wed, 16 Oct 2024 04:19:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a4e050b2-edfc-4534-8f04-6376a3fef961\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:04.237220407Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:04.258180851Z","hostname":"buskin","pid":1572,"sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:93"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:04.26366179Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:38644","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197","component":"producer-server","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:04.26371486Z","hostname":"buskin","pid":1572,"component":"internal-dns-resolver","file":"internal-dns/resolver/src/resolver.rs:111","component":"producer-server","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:04.263769501Z","hostname":"buskin","pid":1572,"file":"oximeter/producer/src/lib.rs:283","component":"producer-server","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/producer/src/lib.rs:283","interval":"30s","address":"[fd00:1122:3344:101::1]:38644","producer_id":"832ac8b0-9619-4005-8cc6-03345e059ddb"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:04.264858215Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"208.285394ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:04.265640785Z","hostname":"buskin","pid":1572,"component":"VmmReservoirManager","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:04.265648995Z","hostname":"buskin","pid":1572,"component":"VmmReservoirManager","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:04.265703896Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:923","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:04.265724306Z","hostname":"buskin","pid":1572,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:04.474368429Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"284.106074ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:04.760944692Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.226769118s"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:05.98979042Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.963559976s"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:06.806446887Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2679"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:06.806472387Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2897"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:06.806477737Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2549"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:06.806495967Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2596"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:06.806553248Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2978"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:06.806559128Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3020"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:06.808411811Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2822"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:06.809681127Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2590"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:06.809789919Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:06.80991567Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 55.549543ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:07.040270085Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 98.559084ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:07.236379971Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 158.197025ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:07.419388222Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 434.440249ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:07.598551344Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:07.954385558Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"4.525270326s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 578.381576ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:08.155981131Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.227796719s","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:08.753576334Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.133646526s","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:10.030093161Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.041176715s","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:11.224937852Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:12.744785439Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"11.555703061s"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:12.750148476Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:12.750205677Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:432"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:12.750231717Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/af85c7e6-e4e0-4385-860c-e4f6171b504b/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:12.750249958Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/76b73db8-8c3d-46ae-bed9-6b12bffa8365/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:12.75047014Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:12.750653212Z","hostname":"buskin","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:12.750723743Z","hostname":"buskin","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.1.100, vlan_id: None, rib_priority: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false, lldp: None }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:12.750744174Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3924"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:12.750748654Z","hostname":"buskin","pid":1572,"remote_addr":"[fdb0:200::1]:45629","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:81"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.005447515Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:117","peer":"[fdb0:200::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.005564766Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:252","target_sled":"[fdb0:200::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.005667537Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/af85c7e6-e4e0-4385-860c-e4f6171b504b/config/rss-service-plan-v4.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.005725718Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/76b73db8-8c3d-46ae-bed9-6b12bffa8365/config/rss-service-plan-v4.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.005737778Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.005748318Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.005769479Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.005781889Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.011703044Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4245","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.011726394Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4261"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.011736024Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:534"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.286214104Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:51571"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.286491948Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.286501998Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/af85c7e6-e4e0-4385-860c-e4f6171b504b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.286565849Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/76b73db8-8c3d-46ae-bed9-6b12bffa8365/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.286585949Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.2866631Z","hostname":"buskin","pid":1572,"uri":"/inventory","method":"GET","req_id":"ad3a993f-4c8c-45db-a23f-31763b36a420","remote_addr":"[fd00:1122:3344:101::1]:51571","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":296,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.317940623Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:38920"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.318094655Z","hostname":"buskin","pid":1572,"uri":"/sled-role","method":"GET","req_id":"c1134cd5-ac85-488a-9e5f-cda1a4e9787f","remote_addr":"[fd00:1122:3344:101::1]:38920","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":37,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.319351111Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/af85c7e6-e4e0-4385-860c-e4f6171b504b/config/.rss-service-plan-v4.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.319376821Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/76b73db8-8c3d-46ae-bed9-6b12bffa8365/config/.rss-service-plan-v4.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.319727326Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.32003014Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:882"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: c67f761d-a822-4cc4-a972-bad9d30bf29e, pool_id: 44b508dd-23c4-4c55-9cb2-5c6030c24fc8 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 1d42f786-f84f-4037-bd04-d6b9eae45a19, pool_id: 7d66fa1b-659d-49d1-9f81-4a613c7223d3 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: e2fb8d20-339c-4f6b-8a71-cc16bb133729, pool_id: 4e27df7e-d6a6-4c23-9bda-dba688b61920 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 872b4234-1d42-472d-acf4-dbe230040c50, pool_id: 01efdee5-e4e9-48c9-bab0-6acb1894267e (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 8bd4f6ca-7b2f-4365-a186-a7ce32f221de, pool_id: 62ca8696-d636-4377-a019-40d38e4ec622 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 6e261974-35d4-4399-aaee-6bc15b503f81, pool_id: f808102a-86a8-4d46-b2ad-f412f245a2e9 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 59ba89cb-8ac6-42ff-8e6c-44dfff1a3b1f, pool_id: 67e52e5a-0417-4522-9270-a4c24dd7c57e (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 487a052c-7aeb-4baf-9a90-57e40ed42c14, pool_id: 61cfa2d7-dade-4fd7-ba85-e2a354f4d8fe (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 4ae0bbf1-be6b-40e5-8788-3ad290790480, pool_id: 6e6a7dcc-a4f5-4b28-9b57-a1e25fa74137 (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.351609307Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:375"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.351702288Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:59130"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.351925421Z","hostname":"buskin","pid":1572,"sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:864"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: c67f761d-a822-4cc4-a972-bad9d30bf29e, pool_id: 44b508dd-23c4-4c55-9cb2-5c6030c24fc8 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 1d42f786-f84f-4037-bd04-d6b9eae45a19, pool_id: 7d66fa1b-659d-49d1-9f81-4a613c7223d3 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: e2fb8d20-339c-4f6b-8a71-cc16bb133729, pool_id: 4e27df7e-d6a6-4c23-9bda-dba688b61920 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 872b4234-1d42-472d-acf4-dbe230040c50, pool_id: 01efdee5-e4e9-48c9-bab0-6acb1894267e (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 8bd4f6ca-7b2f-4365-a186-a7ce32f221de, pool_id: 62ca8696-d636-4377-a019-40d38e4ec622 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 6e261974-35d4-4399-aaee-6bc15b503f81, pool_id: f808102a-86a8-4d46-b2ad-f412f245a2e9 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 59ba89cb-8ac6-42ff-8e6c-44dfff1a3b1f, pool_id: 67e52e5a-0417-4522-9270-a4c24dd7c57e (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 487a052c-7aeb-4baf-9a90-57e40ed42c14, pool_id: 61cfa2d7-dade-4fd7-ba85-e2a354f4d8fe (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 4ae0bbf1-be6b-40e5-8788-3ad290790480, pool_id: 6e6a7dcc-a4f5-4b28-9b57-a1e25fa74137 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.351952941Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/af85c7e6-e4e0-4385-860c-e4f6171b504b/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.351965821Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/76b73db8-8c3d-46ae-bed9-6b12bffa8365/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.351984011Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.351991891Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:866"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.351999042Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.352006692Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.352014882Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.352021932Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.514343563Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:13.514396113Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"44b508dd-23c4-4c55-9cb2-5c6030c24fc8 (zpool)"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/832ac8b0-9619-4005-8cc6-03345e059ddb)","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:13.774741016Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:14.268865148Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:14.268939769Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:14.737357817Z","hostname":"buskin","pid":1572,"component":"VmmReservoirManager","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:14.749365938Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 44b508dd-23c4-4c55-9cb2-5c6030c24fc8 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:14.75742171Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:14.757504511Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:14.757515291Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:14.757536161Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:14.757552521Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:14.757679223Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:14.757689413Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/832ac8b0-9619-4005-8cc6-03345e059ddb)","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:14.776973295Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:14.791283075Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_44b508dd-23c4-4c55-9cb2-5c6030c24fc8/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:14.79884659Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_44b508dd-23c4-4c55-9cb2-5c6030c24fc8","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:14.965968091Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:14.974088822Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:14.974144253Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:14.974163894Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:14.974169454Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:14.974173984Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:14.984865738Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:14.984885078Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"7d66fa1b-659d-49d1-9f81-4a613c7223d3 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.076409679Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 7d66fa1b-659d-49d1-9f81-4a613c7223d3 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.084133536Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.084159196Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.084165377Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.084225967Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.084234757Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.084363099Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.084382969Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.118539469Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_7d66fa1b-659d-49d1-9f81-4a613c7223d3/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.126202655Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_7d66fa1b-659d-49d1-9f81-4a613c7223d3","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.30242777Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.311233351Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.311275762Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.311283762Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.311291312Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.311296802Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.320799961Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.320822702Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"4e27df7e-d6a6-4c23-9bda-dba688b61920 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.41214873Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: 4e27df7e-d6a6-4c23-9bda-dba688b61920 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.419858947Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.419898807Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.419908197Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.419965888Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.419981418Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.420068349Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.42007638Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.454683155Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_4e27df7e-d6a6-4c23-9bda-dba688b61920/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.462708205Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_4e27df7e-d6a6-4c23-9bda-dba688b61920","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.627394106Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.634951111Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.634988191Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.634999542Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.635011572Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.635019092Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.644724944Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.644757114Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"01efdee5-e4e9-48c9-bab0-6acb1894267e (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.735732918Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 01efdee5-e4e9-48c9-bab0-6acb1894267e (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.743400724Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.743442895Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.743458665Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.743498186Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.743518526Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.743677228Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.743690548Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.778366014Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/832ac8b0-9619-4005-8cc6-03345e059ddb)","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:15.781986359Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Automatically destroying dataset: oxp_01efdee5-e4e9-48c9-bab0-6acb1894267e/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.78681293Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_01efdee5-e4e9-48c9-bab0-6acb1894267e","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.962237084Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.970330636Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.970373287Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.970379047Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.970384287Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.970390107Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.980869689Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:15.980903119Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"62ca8696-d636-4377-a019-40d38e4ec622 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.078177052Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 62ca8696-d636-4377-a019-40d38e4ec622 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.086086112Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.086120472Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.086127352Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.086149462Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.086158793Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.086330845Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.086340905Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.121614528Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_62ca8696-d636-4377-a019-40d38e4ec622/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.130159566Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_62ca8696-d636-4377-a019-40d38e4ec622","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.307205941Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.31502769Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.31506579Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.3150738Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.31507934Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.31508378Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.324780253Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.324811823Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"f808102a-86a8-4d46-b2ad-f412f245a2e9 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.421453288Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: f808102a-86a8-4d46-b2ad-f412f245a2e9 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.429348847Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.429372947Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.429379277Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.429402238Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.429410998Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.4296025Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.42961014Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.466075989Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_f808102a-86a8-4d46-b2ad-f412f245a2e9/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.47410493Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_f808102a-86a8-4d46-b2ad-f412f245a2e9","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.651846494Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.659733153Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.659782124Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.659787414Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.659791764Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.659795584Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.669059991Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.669077241Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"67e52e5a-0417-4522-9270-a4c24dd7c57e (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.763921533Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 67e52e5a-0417-4522-9270-a4c24dd7c57e (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.771234615Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.771282626Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.771299286Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.771307576Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.771312336Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.771502959Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.771517529Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/832ac8b0-9619-4005-8cc6-03345e059ddb)","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:16.783632141Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.80743851Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_67e52e5a-0417-4522-9270-a4c24dd7c57e/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.815277039Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_67e52e5a-0417-4522-9270-a4c24dd7c57e","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.987013288Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.99516131Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.995197531Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.995203311Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.995209781Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:16.995215721Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.005303298Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.005337978Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"61cfa2d7-dade-4fd7-ba85-e2a354f4d8fe (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.115576484Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 61cfa2d7-dade-4fd7-ba85-e2a354f4d8fe (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.123096209Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.123130689Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.123137059Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.123167879Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.12317606Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.123326332Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.123335442Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.162047498Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_61cfa2d7-dade-4fd7-ba85-e2a354f4d8fe/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.169736375Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_61cfa2d7-dade-4fd7-ba85-e2a354f4d8fe","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.348810396Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.356838967Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.356863198Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.356869078Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.356882038Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.356886118Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.366434558Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.366463718Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"6e6a7dcc-a4f5-4b28-9b57-a1e25fa74137 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.461580504Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 6e6a7dcc-a4f5-4b28-9b57-a1e25fa74137 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.468593392Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.468622193Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.468655383Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.468692534Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.468700954Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.468848116Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.468855225Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.502056033Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_6e6a7dcc-a4f5-4b28-9b57-a1e25fa74137/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.509187943Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_6e6a7dcc-a4f5-4b28-9b57-a1e25fa74137","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.676311064Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.683294212Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.683327912Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/af85c7e6-e4e0-4385-860c-e4f6171b504b/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.683406633Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.683556965Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: af85c7e6-e4e0-4385-860c-e4f6171b504b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 76b73db8-8c3d-46ae-bed9-6b12bffa8365 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 44b508dd-23c4-4c55-9cb2-5c6030c24fc8 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 7d66fa1b-659d-49d1-9f81-4a613c7223d3 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4e27df7e-d6a6-4c23-9bda-dba688b61920 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 01efdee5-e4e9-48c9-bab0-6acb1894267e (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 62ca8696-d636-4377-a019-40d38e4ec622 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f808102a-86a8-4d46-b2ad-f412f245a2e9 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 67e52e5a-0417-4522-9270-a4c24dd7c57e (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 61cfa2d7-dade-4fd7-ba85-e2a354f4d8fe (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 6e6a7dcc-a4f5-4b28-9b57-a1e25fa74137 (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.683622976Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.683634436Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/76b73db8-8c3d-46ae-bed9-6b12bffa8365/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.683732707Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.68398386Z","hostname":"buskin","pid":1572,"sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:868"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.68399951Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.68400368Z","hostname":"buskin","pid":1572,"sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:894","generation":"Generation(1)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.68401073Z","hostname":"buskin","pid":1572,"sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:899"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.68401573Z","hostname":"buskin","pid":1572,"sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:904"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.684040871Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:121","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.684048731Z","hostname":"buskin","pid":1572,"sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:910"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.684098491Z","hostname":"buskin","pid":1572,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:798","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.684103762Z","hostname":"buskin","pid":1572,"sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:915"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.684186963Z","hostname":"buskin","pid":1572,"uri":"/omicron-physical-disks","method":"PUT","req_id":"b44cf8f2-a497-4305-b0d0-529d032dbfdd","remote_addr":"[fd00:1122:3344:101::1]:59130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4332378,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 1a92457b-f82e-40a6-ba1f-c203f172d56d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 44b508dd-23c4-4c55-9cb2-5c6030c24fc8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b508dd-23c4-4c55-9cb2-5c6030c24fc8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 8aa1c7dd-a278-4e85-84ce-6ec449df30b8 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 7d66fa1b-659d-49d1-9f81-4a613c7223d3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d66fa1b-659d-49d1-9f81-4a613c7223d3 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d299671e-9086-4d15-acd3-9ba8736b40e4 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4e27df7e-d6a6-4c23-9bda-dba688b61920 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e27df7e-d6a6-4c23-9bda-dba688b61920 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.715285084Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.715418255Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:34915"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1a92457b-f82e-40a6-ba1f-c203f172d56d, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b508dd-23c4-4c55-9cb2-5c6030c24fc8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.715716529Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.715725509Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1a92457b-f82e-40a6-ba1f-c203f172d56d, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b508dd-23c4-4c55-9cb2-5c6030c24fc8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8aa1c7dd-a278-4e85-84ce-6ec449df30b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d66fa1b-659d-49d1-9f81-4a613c7223d3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.76353475Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.763564041Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8aa1c7dd-a278-4e85-84ce-6ec449df30b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d66fa1b-659d-49d1-9f81-4a613c7223d3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/832ac8b0-9619-4005-8cc6-03345e059ddb)","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:17.785728539Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d299671e-9086-4d15-acd3-9ba8736b40e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e27df7e-d6a6-4c23-9bda-dba688b61920 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.813023752Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.813054073Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d299671e-9086-4d15-acd3-9ba8736b40e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e27df7e-d6a6-4c23-9bda-dba688b61920 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.86052923Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/af85c7e6-e4e0-4385-860c-e4f6171b504b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.86055614Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/76b73db8-8c3d-46ae-bed9-6b12bffa8365/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.86056257Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.860589901Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.902103282Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.902138643Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.902144313Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_1a92457b-f82e-40a6-ba1f-c203f172d56d","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.95524808Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_8aa1c7dd-a278-4e85-84ce-6ec449df30b8","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.959288371Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_d299671e-9086-4d15-acd3-9ba8736b40e4","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.963469673Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_internal_dns_1a92457b-f82e-40a6-ba1f-c203f172d56d","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.976660739Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_8aa1c7dd-a278-4e85-84ce-6ec449df30b8","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.98156305Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_d299671e-9086-4d15-acd3-9ba8736b40e4","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:17.986025287Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/832ac8b0-9619-4005-8cc6-03345e059ddb)","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:18.788210142Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Profile for oxz_internal_dns_1a92457b-f82e-40a6-ba1f-c203f172d56d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:19.560963246Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:19.561097358Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Booting oxz_internal_dns_1a92457b-f82e-40a6-ba1f-c203f172d56d zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:19.56132916Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_1a92457b-f82e-40a6-ba1f-c203f172d56d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_internal_dns_8aa1c7dd-a278-4e85-84ce-6ec449df30b8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:20.046405308Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:20.526304811Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_d299671e-9086-4d15-acd3-9ba8736b40e4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:20.526380872Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:20.526453563Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"wait for service Some(\"oxz_internal_dns_1a92457b-f82e-40a6-ba1f-c203f172d56d\") failed: Property not found. retry in 29.74103ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:20.535459856Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_1a92457b-f82e-40a6-ba1f-c203f172d56d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_internal_dns_8aa1c7dd-a278-4e85-84ce-6ec449df30b8 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:20.535503417Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_8aa1c7dd-a278-4e85-84ce-6ec449df30b8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_internal_dns_d299671e-9086-4d15-acd3-9ba8736b40e4 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:20.538631456Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d299671e-9086-4d15-acd3-9ba8736b40e4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_internal_dns_1a92457b-f82e-40a6-ba1f-c203f172d56d\") failed: Property not found. retry in 122.741707ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:20.575023213Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_1a92457b-f82e-40a6-ba1f-c203f172d56d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8aa1c7dd-a278-4e85-84ce-6ec449df30b8\") failed: Property not found. retry in 59.930563ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:20.660985704Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_8aa1c7dd-a278-4e85-84ce-6ec449df30b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d299671e-9086-4d15-acd3-9ba8736b40e4\") failed: Property not found. retry in 49.988958ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:20.668196855Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d299671e-9086-4d15-acd3-9ba8736b40e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1a92457b-f82e-40a6-ba1f-c203f172d56d\") failed: Property not found. retry in 167.854504ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:20.707443728Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_1a92457b-f82e-40a6-ba1f-c203f172d56d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d299671e-9086-4d15-acd3-9ba8736b40e4\") failed: Property not found. retry in 123.143327ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:20.726037262Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d299671e-9086-4d15-acd3-9ba8736b40e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8aa1c7dd-a278-4e85-84ce-6ec449df30b8\") failed: Property not found. retry in 74.854162ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:20.847066674Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_8aa1c7dd-a278-4e85-84ce-6ec449df30b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d299671e-9086-4d15-acd3-9ba8736b40e4\") failed: Property not found. retry in 121.25788ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:20.883935487Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d299671e-9086-4d15-acd3-9ba8736b40e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1a92457b-f82e-40a6-ba1f-c203f172d56d\") failed: Property not found. retry in 334.26614ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:20.946069728Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_1a92457b-f82e-40a6-ba1f-c203f172d56d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8aa1c7dd-a278-4e85-84ce-6ec449df30b8\") failed: Property not found. retry in 239.148627ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:21.008352421Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_8aa1c7dd-a278-4e85-84ce-6ec449df30b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d299671e-9086-4d15-acd3-9ba8736b40e4\") failed: Property not found. retry in 280.174341ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:21.046173417Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d299671e-9086-4d15-acd3-9ba8736b40e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8aa1c7dd-a278-4e85-84ce-6ec449df30b8\") failed: Property not found. retry in 417.217089ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:21.25720854Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_8aa1c7dd-a278-4e85-84ce-6ec449df30b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1a92457b-f82e-40a6-ba1f-c203f172d56d\") failed: Property not found. retry in 645.598868ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:21.288626455Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_1a92457b-f82e-40a6-ba1f-c203f172d56d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d299671e-9086-4d15-acd3-9ba8736b40e4\") failed: Property not found. retry in 790.110172ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:21.339745827Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d299671e-9086-4d15-acd3-9ba8736b40e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8aa1c7dd-a278-4e85-84ce-6ec449df30b8\") failed: Property not found. retry in 773.376849ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:21.684208248Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_8aa1c7dd-a278-4e85-84ce-6ec449df30b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1a92457b-f82e-40a6-ba1f-c203f172d56d\") failed: Property not found. retry in 1.132690366s","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:21.943284355Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_1a92457b-f82e-40a6-ba1f-c203f172d56d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d299671e-9086-4d15-acd3-9ba8736b40e4\") failed: Property not found. retry in 1.034367345s","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:22.142196475Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d299671e-9086-4d15-acd3-9ba8736b40e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8aa1c7dd-a278-4e85-84ce-6ec449df30b8\") failed: Property not found. retry in 1.372999713s","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:22.468803231Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_8aa1c7dd-a278-4e85-84ce-6ec449df30b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:22.752040802Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15715154067003207525"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:22.752151853Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"7366151044836788085"} {"msg":"wait for service Some(\"oxz_internal_dns_1a92457b-f82e-40a6-ba1f-c203f172d56d\") failed: Property not found. retry in 717.475232ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:23.087100123Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_1a92457b-f82e-40a6-ba1f-c203f172d56d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:23.100395211Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4269"} {"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:23.100412171Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4279","gateway":"fd00:1122:3344:101::1"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:23.106418726Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4358"} {"msg":"wait for service Some(\"oxz_internal_dns_d299671e-9086-4d15-acd3-9ba8736b40e4\") failed: Property not found. retry in 988.585744ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:23.186407582Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d299671e-9086-4d15-acd3-9ba8736b40e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:23.212561081Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4406"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:23.212596941Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4433"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:23.251611772Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4446"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:23.251644012Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4310"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:23.304208313Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4352"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:23.304236233Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4413","rack_subnet":"fd00:1122:3344::"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:23.323478985Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4424"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:23.323512305Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4466"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:23.389466665Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4498"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:23.389503845Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4504"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:23.475560617Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4545"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:23.475611998Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:370"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/832ac8b0-9619-4005-8cc6-03345e059ddb)","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:23.507797832Z","hostname":"buskin","pid":1572,"component":"NexusNotifierTask","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"Failed to get switch ID from MGS (retrying in 42.227723ms)","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:23.507894914Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:389","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)"} {"msg":"Failed to get switch ID from MGS (retrying in 55.798385ms)","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:23.551053646Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:389","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)"} {"msg":"Failed to get switch ID from MGS (retrying in 291.350385ms)","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:23.608188595Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:389","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)"} {"msg":"wait for service Some(\"oxz_internal_dns_1a92457b-f82e-40a6-ba1f-c203f172d56d\") failed: Property not found. retry in 1.420057144s","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:23.813620377Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_1a92457b-f82e-40a6-ba1f-c203f172d56d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8aa1c7dd-a278-4e85-84ce-6ec449df30b8\") failed: Property not found. retry in 1.033622507s","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:23.849276355Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_8aa1c7dd-a278-4e85-84ce-6ec449df30b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Failed to get switch ID from MGS (retrying in 306.276522ms)","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:23.901568883Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:389","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)"} {"msg":"wait for service Some(\"oxz_internal_dns_d299671e-9086-4d15-acd3-9ba8736b40e4\") failed: Property not found. retry in 770.466062ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:24.184375938Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d299671e-9086-4d15-acd3-9ba8736b40e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.203782772Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::2]:47615"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.203990505Z","hostname":"buskin","pid":1572,"uri":"/sled-identifiers","method":"GET","req_id":"31a3ab83-4469-4779-a9c2-c20ba59eba55","remote_addr":"[fd00:1122:3344:101::2]:47615","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":64,"response_code":"200"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.209874309Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:418"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.242134374Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:686"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.242237366Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:724"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.24262023Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:727","uptime":"0"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.24263668Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:440","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:24.276118812Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:24.276251133Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.278460701Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.330810389Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.33087387Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.33088557Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.33089494Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.3309017Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.33091053Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.33091811Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.330924761Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.330950661Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.330957841Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.330964681Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.330972901Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.330979361Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.330985901Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.330990671Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.330995511Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.331001902Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.331007042Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.331012072Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.331016942Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.331025612Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.331030532Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:24.331035252Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"wait for service Some(\"oxz_internal_dns_8aa1c7dd-a278-4e85-84ce-6ec449df30b8\") failed: Property not found. retry in 1.279077829s","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:24.892660423Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_8aa1c7dd-a278-4e85-84ce-6ec449df30b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d299671e-9086-4d15-acd3-9ba8736b40e4\") failed: Property not found. retry in 579.495323ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:24.963161338Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d299671e-9086-4d15-acd3-9ba8736b40e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.190919122Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_internal_dns_1a92457b-f82e-40a6-ba1f-c203f172d56d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.190963113Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_internal_dns_8aa1c7dd-a278-4e85-84ce-6ec449df30b8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.190971173Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_internal_dns_d299671e-9086-4d15-acd3-9ba8736b40e4"} {"msg":"Writing ledger to /pool/int/af85c7e6-e4e0-4385-860c-e4f6171b504b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.190983153Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/76b73db8-8c3d-46ae-bed9-6b12bffa8365/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.191304827Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.19156212Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"c0e428c1-eb42-4373-a999-4df95525cb94","remote_addr":"[fd00:1122:3344:101::1]:34915","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8476043,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.193605686Z","hostname":"buskin","pid":1572,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.245711921Z","hostname":"buskin","pid":1572,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.291827331Z","hostname":"buskin","pid":1572,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.338758041Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:611"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.338770941Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.371633964Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.372450265Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.372458415Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 1a92457b-f82e-40a6-ba1f-c203f172d56d (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 44b508dd-23c4-4c55-9cb2-5c6030c24fc8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b508dd-23c4-4c55-9cb2-5c6030c24fc8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 8aa1c7dd-a278-4e85-84ce-6ec449df30b8 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 7d66fa1b-659d-49d1-9f81-4a613c7223d3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d66fa1b-659d-49d1-9f81-4a613c7223d3 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d299671e-9086-4d15-acd3-9ba8736b40e4 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 4e27df7e-d6a6-4c23-9bda-dba688b61920 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e27df7e-d6a6-4c23-9bda-dba688b61920 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 94215135-ca88-480e-825c-ad8c4debf0e5 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 6e6a7dcc-a4f5-4b28-9b57-a1e25fa74137 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 0a097c51-c57e-47d4-8657-551d990d96e5, kind: Service { id: 94215135-ca88-480e-825c-ad8c4debf0e5 }, name: Name(\"ntp-94215135-ca88-480e-825c-ad8c4debf0e5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 218, 170])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.40153968Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.401653082Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:62187"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1a92457b-f82e-40a6-ba1f-c203f172d56d, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b508dd-23c4-4c55-9cb2-5c6030c24fc8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.401911245Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.401924395Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1a92457b-f82e-40a6-ba1f-c203f172d56d, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b508dd-23c4-4c55-9cb2-5c6030c24fc8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8aa1c7dd-a278-4e85-84ce-6ec449df30b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d66fa1b-659d-49d1-9f81-4a613c7223d3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.420964384Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.421001845Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8aa1c7dd-a278-4e85-84ce-6ec449df30b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d66fa1b-659d-49d1-9f81-4a613c7223d3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d299671e-9086-4d15-acd3-9ba8736b40e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e27df7e-d6a6-4c23-9bda-dba688b61920 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.438675537Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.438709347Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d299671e-9086-4d15-acd3-9ba8736b40e4, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e27df7e-d6a6-4c23-9bda-dba688b61920 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.456365549Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/af85c7e6-e4e0-4385-860c-e4f6171b504b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.45640448Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/76b73db8-8c3d-46ae-bed9-6b12bffa8365/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.456598892Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.456723504Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.467746523Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.50015874Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.500820319Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.500836729Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.530251738Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 218, 170]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.530267189Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.530633943Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.530641643Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.530914737Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.531307412Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.554681845Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:26.572486849Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:27.205673589Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:27.2057407Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5\") failed: Property not found. retry in 56.205941ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:27.335930546Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5\") failed: Property not found. retry in 86.655837ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:27.403185802Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5\") failed: Property not found. retry in 166.017158ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:27.522146498Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5\") failed: Property not found. retry in 286.329879ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:27.69895739Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5\") failed: Property not found. retry in 939.211448ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:27.993980409Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5\") failed: Property not found. retry in 1.102876945s","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:28.942426032Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5\") failed: Property not found. retry in 1.166495021s","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:30.054075346Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5\") failed: Property not found. retry in 1.316166529s","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:31.231804632Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:32.569055743Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5"} {"msg":"Writing ledger to /pool/int/af85c7e6-e4e0-4385-860c-e4f6171b504b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:32.569135514Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/76b73db8-8c3d-46ae-bed9-6b12bffa8365/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:32.569481929Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:32.569791213Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"2f88135a-b16b-4d1a-8a31-2bc7c4009ebf","remote_addr":"[fd00:1122:3344:101::1]:62187","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":6168050,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:32.572278764Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:32.602378392Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:32.602512354Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:39545"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:32.611468277Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3877"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:32.611589978Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"5f1a94ff-6b8d-4482-84f8-200c4e39bd2d","remote_addr":"[fd00:1122:3344:101::1]:39545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8919,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:32.614087769Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:32.708113921Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:32.708282364Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:51436"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:32.754201271Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15715154067003207525"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:32.754326982Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"7366151044836788085"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:34.278639384Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:34.278763386Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:35.260129393Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9400442939787624856"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:35.567536057Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6803793857447086899"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:36.19376491Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"16293510612634613239"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:36.717250341Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3877"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:36.717423254Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"843d871e-2a93-4de5-9a84-43ab2eb3dfaa","remote_addr":"[fd00:1122:3344:101::1]:51436","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009039,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:36.719367618Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:36.81660651Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:36.816784033Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:34751"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:36.82535774Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3877"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:36.825447212Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"bfc287ab-0cc7-4746-b199-95bbd1f32ff3","remote_addr":"[fd00:1122:3344:101::1]:34751","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8580,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:36.827218274Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:37.048964531Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:37.049115043Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:52568"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_94215135-ca88-480e-825c-ad8c4debf0e5': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:37.05682315Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3877"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:37.056962171Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"52afd301-eb3b-402c-8d09-0681916ece37","remote_addr":"[fd00:1122:3344:101::1]:52568","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":7796,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:37.059975499Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:37.239254053Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:37.239361364Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:48231"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:41.247919016Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"389d3b57-6357-4521-877a-326d4457520e","remote_addr":"[fd00:1122:3344:101::1]:48231","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4008477,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:41.250742382Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:41.250756812Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:41.464864414Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:41.464995665Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:55226"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:41.473626434Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"b97459df-eb19-4d8e-8398-5be9794e0631","remote_addr":"[fd00:1122:3344:101::1]:55226","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8559,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:41.476915595Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:41.476924065Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:41.785752088Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:41.785864329Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:56484"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:41.793414164Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"2f1e517d-7001-4ec2-9e1a-2ec20fe8859c","remote_addr":"[fd00:1122:3344:101::1]:56484","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7487,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:41.796747776Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:41.796758206Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:42.168102255Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:42.168195366Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:41059"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:42.17731484Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"5c51771b-4e27-4431-bf2f-a61912b37a89","remote_addr":"[fd00:1122:3344:101::1]:41059","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8962,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:42.179086403Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:42.179092903Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:42.573706684Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17547664173444279593"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:42.573883496Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6018873948410444826"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:42.693762763Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:42.693942505Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:51753"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:42.702700165Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"e84b097e-c9e1-4410-84f8-cdab1172001c","remote_addr":"[fd00:1122:3344:101::1]:51753","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8663,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:42.704974774Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:42.704984804Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:42.756135177Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15715154067003207525"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:42.756255779Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"7366151044836788085"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:42.969394977Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:42.969533159Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:62964"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:42.978115847Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"0f16d186-c186-43f3-a60d-166a6177bf34","remote_addr":"[fd00:1122:3344:101::1]:62964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8527,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:42.979846749Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:42.979852329Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:43.6457216Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:43.645827042Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:56770"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:43.65445441Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"45832a91-c57b-43e8-93fe-1317081521be","remote_addr":"[fd00:1122:3344:101::1]:56770","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8468,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:43.655589754Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:43.655594364Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:44.213083012Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:44.213237494Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:38519"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:44.221368656Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"17cdf2f6-2798-45b8-9088-cc1884fc60f6","remote_addr":"[fd00:1122:3344:101::1]:38519","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8062,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:44.225295456Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:44.225300686Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:44.281787616Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:44.281907557Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:44.341176922Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"284.049689ms"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:45.262044539Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9400442939787624856"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:45.324153039Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:45.324291451Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:51670"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:45.33298187Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"293dc0bb-319a-4511-83bb-8f5fcadca784","remote_addr":"[fd00:1122:3344:101::1]:51670","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8610,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:45.336044969Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:45.336055159Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:45.57080284Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6803793857447086899"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:45.95972671Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:45.959872932Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:58332"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:45.967818161Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"1d94d929-fb0d-469c-a21a-e36d306738b1","remote_addr":"[fd00:1122:3344:101::1]:58332","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7869,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:45.970335712Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:45.970344253Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:46.195961269Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"16293510612634613239"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:46.743500873Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:46.743643804Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:36597"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:46.752420835Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"0ad8df22-36aa-496d-8575-f252212b36dd","remote_addr":"[fd00:1122:3344:101::1]:36597","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8703,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:46.754482141Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:46.754491131Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:47.749944394Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:47.750110717Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:37471"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:47.758471742Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"b4793241-da36-4552-8115-fa4be803c5a4","remote_addr":"[fd00:1122:3344:101::1]:37471","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8275,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:47.762169128Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:47.762186628Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:49.317030074Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:49.317141135Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:48537"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:49.326103078Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"ba080330-a767-4b82-90ec-6ab16677ed40","remote_addr":"[fd00:1122:3344:101::1]:48537","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8843,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:49.329771214Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:49.329776674Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:50.357742517Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:50.357885409Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:62903"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:50.366667529Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"981976fb-7303-40cc-b8af-df2d901833a4","remote_addr":"[fd00:1122:3344:101::1]:62903","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8686,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:50.370754671Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:50.370762771Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:51.673920523Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:51.674038724Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:39596"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:51.682204627Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"fcff3798-3ba6-497c-94ad-ee1d7e5520a8","remote_addr":"[fd00:1122:3344:101::1]:39596","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8002,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:51.685224595Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:51.685233225Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:52.576025093Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17547664173444279593"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:52.576144055Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6018873948410444826"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:52.757409194Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15715154067003207525"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:52.757555295Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"7366151044836788085"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:53.280673002Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:200::2]:57224"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:53.280889244Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"f623bf0e-fe77-4375-8bd4-4355f5b058e8","remote_addr":"[fdb0:200::2]:57224","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":81,"response_code":"200"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:53.563449207Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:53.563614399Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:46059"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:53.572734644Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"482daf93-e5c9-48cd-a95e-838a638e1cf7","remote_addr":"[fd00:1122:3344:101::1]:46059","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":9045,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:53.575688161Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:53.575698491Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:54.284547732Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:54.284660303Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:55.245453961Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:55.245642343Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:59983"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:55.254781738Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"b34fb1fa-3605-4091-acf5-e285ae19f97d","remote_addr":"[fd00:1122:3344:101::1]:59983","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":9063,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:55.258739258Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:55.258749428Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:55.264123736Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9400442939787624856"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:55.572499193Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6803793857447086899"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:19:56.197329568Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"16293510612634613239"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:57.264740537Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:57.264921719Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:58079"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:57.274743212Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"db090fec-7cfe-490e-9963-78f0746d7586","remote_addr":"[fd00:1122:3344:101::1]:58079","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":9726,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:19:57.277840201Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:57.277850351Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:19:59.660178941Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"521.600539ms"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:02.305246603Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:02.305446365Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:34266"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:02.315201648Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"bef5e8cc-ae7d-4e58-bc35-867769d2c064","remote_addr":"[fd00:1122:3344:101::1]:34266","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":9661,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:02.317947502Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:20:02.317964863Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:02.578597549Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17547664173444279593"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:02.578716121Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6018873948410444826"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:02.757996745Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15715154067003207525"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:02.759125419Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"7366151044836788085"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:04.287071967Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:04.28729315Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:05.265270974Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9400442939787624856"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:05.574512392Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6803793857447086899"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:06.198773829Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"16293510612634613239"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:10.008197937Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:10.008370049Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:34884"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:10.018012991Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"9a5527cd-c215-408a-a2dc-89c79b47b96f","remote_addr":"[fd00:1122:3344:101::1]:34884","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":9545,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:10.020570433Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:20:10.020578563Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:12.580816789Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17547664173444279593"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:12.580934551Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6018873948410444826"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:12.759163131Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"15715154067003207525"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:12.759766539Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"7366151044836788085"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:13.426713883Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:13.426964406Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:43957"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:13.436681578Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"8a3e283a-cad4-4042-8d31-b70773288771","remote_addr":"[fd00:1122:3344:101::1]:43957","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":9602,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:13.43920294Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:20:13.4392243Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:14.288991282Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:14.289104394Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:20:15.217096621Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"779.303022ms"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:15.26720279Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9400442939787624856"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:15.57655654Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6803793857447086899"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:16.201269123Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"16293510612634613239"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:21.23697924Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:21.237154962Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:46422"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:21.246348297Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"93333a55-222c-4f39-af30-0cf48229f36d","remote_addr":"[fd00:1122:3344:101::1]:46422","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":9110,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:21.24899042Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:20:21.249015711Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:22.582706967Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17547664173444279593"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:22.582828748Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6018873948410444826"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:23.315346777Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:200::2]:46616"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:23.315460668Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"90c662d9-a5ea-4f06-babc-47ccaf58bf6f","remote_addr":"[fdb0:200::2]:46616","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":40,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:24.291575859Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:24.291783651Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:25.268951075Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"9400442939787624856"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:25.578544287Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"6803793857447086899"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:26.202864766Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"16293510612634613239"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:20:31.030217851Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"1.449957964s"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:32.585640055Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"17547664173444279593"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:32.585868138Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"6018873948410444826"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:34.040317522Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:34.040501794Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:41079"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:34.049550738Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"afdfccbb-c6c1-484f-9a90-ad735baadcc3","remote_addr":"[fd00:1122:3344:101::1]:41079","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8946,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:34.052647007Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:20:34.052652377Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:34.29387546Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:34.293995481Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:44.296121732Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:44.296261724Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:20:47.514178286Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"3.134150097s"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:51.088757733Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:51.088899295Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:64490"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:51.097983669Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"84cc4a03-f89a-4f05-816d-600a498a83fc","remote_addr":"[fd00:1122:3344:101::1]:64490","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":9006,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:51.10126466Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:20:51.10127537Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:53.347322316Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:200::2]:39465"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:20:53.347433877Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"526acfad-5339-4be0-aa76-d6fa6c0c63a8","remote_addr":"[fdb0:200::2]:39465","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":39,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:54.298415173Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:20:54.298532934Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:21:01.757028307Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:21:01.757224389Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:53162"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:21:01.765829347Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"e0985a2b-ab05-4638-986d-1bbeaa22649b","remote_addr":"[fd00:1122:3344:101::1]:53162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8506,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:21:01.768629562Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:21:01.768634902Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:21:04.299895214Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:21:04.300017175Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:21:05.683590078Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"6.128745588s"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:21:14.302392557Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:21:14.30265199Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:21:21.031280156Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:21:21.031410848Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:40061"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:21:21.039915815Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"4b1d63be-0238-4ac7-a354-3b899949aa09","remote_addr":"[fd00:1122:3344:101::1]:40061","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8441,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:21:21.04266325Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:21:21.04267123Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:21:23.379607678Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:200::2]:33442"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:21:23.37972252Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"5cd61a11-61cc-4816-85a5-42779eb40e76","remote_addr":"[fdb0:200::2]:33442","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":42,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:21:24.303748146Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:21:24.304036859Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:21:26.846098796Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"11.581547109s"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:21:34.305797674Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:21:34.305980376Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:21:43.446189961Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:21:43.446320132Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:46320"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:21:43.454928001Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"92aed828-ea54-433d-aac8-f95651b42c07","remote_addr":"[fd00:1122:3344:101::1]:46320","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8516,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:21:43.45888502Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:21:43.45889218Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:21:44.308692163Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:21:44.308803624Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:21:53.411189043Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:200::2]:38594"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:21:53.411314274Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"a9cd6102-d1ac-48ce-8d29-815df01849c6","remote_addr":"[fdb0:200::2]:38594","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":45,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:21:53.46186699Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"34.831010881s"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:21:54.31138015Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:21:54.311659653Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:21:59.943491913Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:21:59.943611194Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:44024"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:21:59.953330766Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"c6789f39-9212-46bb-9b24-7745b22feb90","remote_addr":"[fd00:1122:3344:101::1]:44024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":9680,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:21:59.958233438Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:21:59.958245748Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:22:04.313080834Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:22:04.313186905Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:22:14.314720188Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:22:14.314986591Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:22:23.444443791Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:200::2]:35401"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:22:23.444542842Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"357fa495-4461-47d4-bd61-e72c99330ecd","remote_addr":"[fdb0:200::2]:35401","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":38,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:22:24.317181882Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:22:24.317300453Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:22:32.779316462Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:22:32.779454133Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:60281"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:22:32.78871812Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"a62f081f-bc97-453f-90d3-1ce521f6c28a","remote_addr":"[fd00:1122:3344:101::1]:60281","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":9221,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:22:32.793575551Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:22:32.793582771Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:22:34.319725507Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:22:34.319839118Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:22:43.329161477Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"86.027187289s"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:22:44.321788136Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:22:44.321952948Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:22:53.479352218Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:200::2]:41480"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:22:53.479457309Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"ae523b8e-e4e3-49bf-8659-5c0af41fd816","remote_addr":"[fdb0:200::2]:41480","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":39,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:22:54.324246289Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:22:54.32435852Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:23:04.326607181Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:23:04.326709232Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:23:12.812256577Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:23:12.812380568Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:47171"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:23:12.821451622Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"3d62eaa5-cac7-42e2-b4e4-3fa83ba9004a","remote_addr":"[fd00:1122:3344:101::1]:47171","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":9002,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:23:12.824574732Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:23:12.824584812Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:23:14.328825392Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:23:14.328938824Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:23:23.511303867Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:200::2]:60240"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:23:23.511418329Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"50d5da28-79c2-4af8-8627-53fbea26fc81","remote_addr":"[fdb0:200::2]:60240","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":43,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:23:24.331483858Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:23:24.331597319Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:23:34.334533379Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:23:34.334795522Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:23:44.33687088Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:23:44.336981132Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:23:53.544767365Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:200::2]:48154"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:23:53.544821136Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"142a0af2-ac88-4473-ba4a-b21d776dcf79","remote_addr":"[fdb0:200::2]:48154","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":26,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:23:54.338570814Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:23:54.338685485Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:24:00.260261716Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:24:00.260450748Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:39305"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:24:00.269747125Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"f2d10040-d7e9-45d0-99fd-9127d1eb9472","remote_addr":"[fd00:1122:3344:101::1]:39305","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":9198,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:24:00.272681772Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:24:00.272688632Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:24:04.340991836Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:24:04.341092327Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:24:14.342743572Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:24:14.342848063Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:24:17.684615243Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:24:23.576297029Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:200::2]:41542"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:24:23.576427431Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"41f3cbf7-5cb3-4b5c-861c-26f12ceca27f","remote_addr":"[fdb0:200::2]:41542","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":40,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:24:24.345168335Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:24:24.345280916Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:24:24.391840141Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"126.115247773s"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:24:34.347270693Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:24:34.347371484Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:24:35.027713697Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:24:35.027851538Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:40449"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:24:35.036862472Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"1fa0bdf0-c007-48a6-a352-ec7ad456be6e","remote_addr":"[fd00:1122:3344:101::1]:40449","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8940,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:24:35.042155458Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:24:35.042163078Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:24:44.348616633Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:24:44.348717604Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:24:53.609628266Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:200::2]:49245"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:24:53.609687776Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"38fccbac-8943-41a8-a4cb-17464ff55133","remote_addr":"[fdb0:200::2]:49245","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":33,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:24:54.351577603Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:24:54.351687504Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:25:04.353906936Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:25:04.354090378Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:25:06.246091142Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:25:06.246216724Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:43850"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:25:06.255102585Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"48204345-b51a-477e-b056-84597ed0266f","remote_addr":"[fd00:1122:3344:101::1]:43850","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8816,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:25:06.25787017Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:25:06.25788063Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:25:14.356292208Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:25:14.356391949Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:25:23.64119079Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:200::2]:62397"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:25:23.641260351Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"5cdeed0d-d628-4379-92c5-9dfcc8052795","remote_addr":"[fdb0:200::2]:62397","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":32,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:25:24.358465707Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:25:24.358597488Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:25:34.360484766Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:25:34.360590708Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:25:44.362129931Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:25:44.362234132Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:25:53.674055903Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:200::2]:38977"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:25:53.674128584Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"35c9bcb2-f54b-47f7-a444-a084e20e36fb","remote_addr":"[fdb0:200::2]:38977","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":34,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:25:54.363958996Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:25:54.364190389Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:26:04.366242357Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:26:04.36644314Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:26:07.783161992Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:26:07.783288074Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:37341"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:26:07.792011214Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"5bbcd10a-4d62-4b8c-b528-b9d2525ada09","remote_addr":"[fd00:1122:3344:101::1]:37341","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8664,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:26:07.796660462Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:26:07.796668322Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:26:14.367852371Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:26:14.367954802Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:26:23.70905079Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:200::2]:61081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:26:23.709159212Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"0f03fc29-5386-4790-815b-9856d685109e","remote_addr":"[fdb0:200::2]:61081","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":38,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:26:24.370186991Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:26:24.370310323Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:26:34.372070927Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:26:34.372173558Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:26:44.374195406Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:26:44.374295517Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:26:45.541016374Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers)\"","delay":"217.662506572s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:26:53.741858368Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:200::2]:64669"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:26:53.741897328Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"751424cc-7851-4c2c-a1cd-93bd7a172bdd","remote_addr":"[fdb0:200::2]:64669","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":26,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:26:54.376436475Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:26:54.376570297Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:27:04.377931475Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:27:04.378043296Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:27:14.380311657Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:27:14.380418039Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:27:23.77557549Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:200::2]:48659"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:27:23.77563689Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"4235f5bd-8ee8-434f-a139-fced98bdd9a6","remote_addr":"[fdb0:200::2]:48659","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":35,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:27:24.381527137Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:27:24.38176231Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:27:28.11968379Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:27:28.119812241Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:36624"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:27:28.129132089Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"e7982222-eaac-49f4-8daf-e1f8e8940aed","remote_addr":"[fd00:1122:3344:101::1]:36624","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":9251,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:27:28.132135006Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-16T04:27:28.132169557Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:27:34.384247963Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:27:34.384355114Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:27:44.386124649Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:27:44.386325251Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:27:53.808015995Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:200::2]:61483"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-16T04:27:53.808083746Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"2d67b022-250f-46b6-9d2f-b80622f59125","remote_addr":"[fdb0:200::2]:61483","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":36,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:27:54.387702153Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:27:54.387908305Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:28:04.389748931Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:28:04.389859273Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:28:14.392403187Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10068038140855432048"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-16T04:28:14.392503409Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"832ac8b0-9619-4005-8cc6-03345e059ddb","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"364730678587034324"}