[ Feb 14 20:02:31 Enabled. ] [ Feb 14 20:02:31 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Feb 14 20:02:31 Method "start" exited with status 0. ] [ Feb 14 20:02:31 Rereading configuration. ] [ Feb 14 20:02:31 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:31.712066009Z","hostname":"buskin","pid":1566,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:31.71222079Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:31.764340048Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:31.764376588Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:31.878462296Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:31.888193326Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:31.896616084Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:32.501227453Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:9fb2::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:32.640474621Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:126"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:32.640519351Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:138"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:32.640579172Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:151"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:32.640655143Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:32.640670613Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:32.640708723Z","hostname":"buskin","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:717"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:32.643660694Z","hostname":"buskin","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:624"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:32.643667184Z","hostname":"buskin","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:766"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:32.643681394Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:187"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:32.643686314Z","hostname":"buskin","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:825"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:32.643695574Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:32.643754885Z","hostname":"buskin","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:32.643866586Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:32.643897716Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:32.643910206Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:32.655407305Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:32.655442975Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:32.752260715Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.11712765Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.117160561Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.117166761Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.117179211Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.117191141Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.117204861Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.117209901Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.12578221Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.12581206Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"26a6a25d-ab9a-45d6-baed-89b7374cc198 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.211293202Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 26a6a25d-ab9a-45d6-baed-89b7374cc198 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564363986Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564417647Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 26a6a25d-ab9a-45d6-baed-89b7374cc198 (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564467467Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564478647Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564489367Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564496827Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564501058Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564507278Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564512208Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564518258Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564522938Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 26a6a25d-ab9a-45d6-baed-89b7374cc198 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564543848Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564549818Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564555038Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564561398Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564566378Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 26a6a25d-ab9a-45d6-baed-89b7374cc198 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564586108Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564591528Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564595479Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564599739Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 26a6a25d-ab9a-45d6-baed-89b7374cc198 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564618189Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564626949Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564632019Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564637609Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564641469Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564645469Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 26a6a25d-ab9a-45d6-baed-89b7374cc198 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564665309Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564670969Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.5646757Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564681689Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564686509Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564690669Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 26a6a25d-ab9a-45d6-baed-89b7374cc198 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.56471395Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.56471901Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.56472734Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.56473169Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.56473685Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.56474345Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.56474895Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.56475458Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 26a6a25d-ab9a-45d6-baed-89b7374cc198 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.56478637Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564791451Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564795061Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564799581Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564803381Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564822581Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564826371Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 26a6a25d-ab9a-45d6-baed-89b7374cc198 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564855991Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564861041Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564865111Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564868542Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564872942Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 26a6a25d-ab9a-45d6-baed-89b7374cc198 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564894822Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564903572Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564907132Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:97"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564917902Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:99"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564921422Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 26a6a25d-ab9a-45d6-baed-89b7374cc198 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564944822Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564949482Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564952932Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564956372Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:211"} {"msg":"No ledger in /pool/int/a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.564989183Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/26a6a25d-ab9a-45d6-baed-89b7374cc198/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.565001383Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.565012273Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.565024073Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/26a6a25d-ab9a-45d6-baed-89b7374cc198/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.565031463Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.565035693Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.565066504Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:216"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.565071013Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:231"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.575502531Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:794"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.575554792Z","hostname":"buskin","pid":1566,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999907829s","next_cleanup":"Instant { tv_sec: 1220, tv_nsec: 831389991 }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.575595362Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.575641083Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4067"} {"msg":"No ledger in /pool/int/a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.575647913Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/26a6a25d-ab9a-45d6-baed-89b7374cc198/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.575653893Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.579021648Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:328"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.580841156Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:51","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: Some(\n None,\n ),\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n tx_eq: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.581109829Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1195"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.581115419Z","hostname":"buskin","pid":1566,"component":"DnsResolver","component":"RSS","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.58117656Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.58120497Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617/config/rss-started.marker","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.58124013Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/26a6a25d-ab9a-45d6-baed-89b7374cc198/config/rss-started.marker","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.581266501Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.581278091Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/26a6a25d-ab9a-45d6-baed-89b7374cc198/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.581284981Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS not previously run. Creating plans.","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.581291751Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1244"} {"msg":"Writing ledger to /pool/int/a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617/config/.rss-started.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.581297891Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/26a6a25d-ab9a-45d6-baed-89b7374cc198/config/.rss-started.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.581662525Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.581847547Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1276"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.581860237Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:66"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.581864027Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:39"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.581916347Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1307"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.581975158Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.581988608Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/26a6a25d-ab9a-45d6-baed-89b7374cc198/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.58213638Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.582270221Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.582346252Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: e20b6ac2-4792-45a8-adc6-0eb48d767a22 (sled), rack_id: 111dc2d5-0d13-4794-b553-9a7ef18590ba, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.583213251Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4364"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.583290141Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4816"} {"msg":"Loaded keys and certs","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.587641117Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/client.rs:50"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.593594588Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.59377856Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.598509839Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.598575509Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.59867363Z","hostname":"buskin","pid":1566,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:46815","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:74"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: e20b6ac2-4792-45a8-adc6-0eb48d767a22 (sled), rack_id: 111dc2d5-0d13-4794-b553-9a7ef18590ba, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.598805702Z","hostname":"buskin","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:362"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.598816222Z","hostname":"buskin","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:377"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.598850162Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/server.rs:44"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.598854862Z","hostname":"buskin","pid":1566,"component":"DnsResolver","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.598872972Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.598892253Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:813"} {"msg":"No ledger in /pool/int/a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.598927423Z","hostname":"buskin","pid":1566,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.598934983Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"No ledger in /pool/int/26a6a25d-ab9a-45d6-baed-89b7374cc198/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.598945523Z","hostname":"buskin","pid":1566,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.598956723Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:802"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.598962733Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:830"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.598968493Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:596"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.598977754Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.598986803Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:701","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.616692746Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.637492021Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.655495747Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.680606336Z","hostname":"buskin","pid":1566,"sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:419"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.680650957Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.680654797Z","hostname":"buskin","pid":1566,"sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:431"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.680658277Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 138.243122ms)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.68390202Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dff91f6d-9694-4f93-8e36-580555dcaa24\", \"content-length\": \"124\", \"date\": \"Fri, 14 Feb 2025 20:02:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dff91f6d-9694-4f93-8e36-580555dcaa24\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.694455129Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.71874073Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617/swap\"","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.71877803Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.724151835Z","hostname":"buskin","pid":1566,"sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:446"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.724167326Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:129"} {"msg":"Ensuring dataset oxi_a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.724174976Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:139"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.792922575Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:174"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.823059996Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 103.198498ms)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.823653032Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"11360716-7914-4f7f-858a-6a23e18f11eb\", \"content-length\": \"124\", \"date\": \"Fri, 14 Feb 2025 20:02:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"11360716-7914-4f7f-858a-6a23e18f11eb\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.927730776Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 78.721536ms)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:33.928309553Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"897a6912-2fb4-40a7-bd09-4f76cec983a2\", \"content-length\": \"124\", \"date\": \"Fri, 14 Feb 2025 20:02:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"897a6912-2fb4-40a7-bd09-4f76cec983a2\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:34.007278188Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 182.167487ms)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:34.007740602Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0a08c4b2-7c93-4cfc-8299-86cdd389ced1\", \"content-length\": \"124\", \"date\": \"Fri, 14 Feb 2025 20:02:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0a08c4b2-7c93-4cfc-8299-86cdd389ced1\" }"} {"msg":"Mounting oxi_a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:34.01145806Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:182"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:34.024592296Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:190"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:34.024710347Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:190"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:34.024751218Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:190"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:34.024793258Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:196"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:34.190791922Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 119.768488ms)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:34.191524529Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7466a0c7-ca77-4e3f-a47a-8be6c6c86fdc\", \"content-length\": \"124\", \"date\": \"Fri, 14 Feb 2025 20:02:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7466a0c7-ca77-4e3f-a47a-8be6c6c86fdc\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:34.31269023Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 205.932069ms)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:34.313376227Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ceaeecf1-e653-48bd-85a9-e3c4ac4a4284\", \"content-length\": \"124\", \"date\": \"Fri, 14 Feb 2025 20:02:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ceaeecf1-e653-48bd-85a9-e3c4ac4a4284\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:34.520448774Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 409.454685ms)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:34.52108088Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d8b54a5a-3e5c-4966-bfe3-eff5e2b60c60\", \"content-length\": \"124\", \"date\": \"Fri, 14 Feb 2025 20:02:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d8b54a5a-3e5c-4966-bfe3-eff5e2b60c60\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:34.766219631Z","hostname":"buskin","pid":1566,"sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:93"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:34.772226973Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:47699","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:328","component":"producer-server","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:328"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:34.772253423Z","hostname":"buskin","pid":1566,"component":"internal-dns-resolver","file":"internal-dns/resolver/src/resolver.rs:111","component":"producer-server","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:34.772323074Z","hostname":"buskin","pid":1566,"file":"oximeter/producer/src/lib.rs:279","component":"producer-server","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/producer/src/lib.rs:279","interval":"30s","address":"[fd00:1122:3344:101::1]:47699","producer_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:34.773614027Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"288.788005ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:34.774566517Z","hostname":"buskin","pid":1566,"component":"VmmReservoirManager","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:34.774572557Z","hostname":"buskin","pid":1566,"component":"VmmReservoirManager","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:34.774623277Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:978","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:34.774633317Z","hostname":"buskin","pid":1566,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:34.931630538Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:35.063989804Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"484.342154ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:35.550805438Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"576.696381ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:36.129670122Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.439045132s"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:37.558518049Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2750"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:37.558541779Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2978"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:37.558546869Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2620"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:37.558551269Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2667"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:37.55857842Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3059"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:37.5585862Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3128"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:37.56052595Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2886"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:37.561892854Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2661"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:37.562042025Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:37.562183067Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:489"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_switch\") failed: Property not found. retry in 60.26424ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:37.973212609Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_switch\") failed: Property not found. retry in 73.033598ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:38.201735198Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_switch\") failed: Property not found. retry in 195.564453ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:38.397508078Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:38.570588574Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"5.879177612s"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_switch\") failed: Property not found. retry in 308.080549ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:38.779291228Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_switch\") failed: Property not found. retry in 748.364743ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:39.217245388Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_switch\") failed: Property not found. retry in 770.865551ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:40.099456043Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_switch\") failed: Property not found. retry in 651.561733ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:40.923119774Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_switch\") failed: Property not found. retry in 1.240883221s","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:41.782428233Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_switch\") failed: Property not found. retry in 565.144214ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:43.096719227Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.067100072Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.067488816Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12348","component":"dropshot (Repo Depot)","component":"ArtifactStore","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:328"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.06881101Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:328"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.06884258Z","hostname":"buskin","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:432"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.06885425Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Writing ledger to /pool/int/a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.06886857Z","hostname":"buskin","pid":1566,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/26a6a25d-ab9a-45d6-baed-89b7374cc198/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.069039012Z","hostname":"buskin","pid":1566,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.069233124Z","hostname":"buskin","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.069304365Z","hostname":"buskin","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.1.100, vlan_id: None, rib_priority: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: Some(None), bgp_peers: [], autoneg: false, lldp: None, tx_eq: None }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.069316085Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4067"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.069361315Z","hostname":"buskin","pid":1566,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:46815","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:81"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.26167765Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:117","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.261783691Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:252","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.266999865Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4388","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.267024705Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4404"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.267036016Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:535"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:44.476973763Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"7.002223906s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.503383795Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:36766"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.503654768Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.503663608Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.503744419Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/26a6a25d-ab9a-45d6-baed-89b7374cc198/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.503772089Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.50388496Z","hostname":"buskin","pid":1566,"uri":"/inventory","method":"GET","req_id":"6c823cd8-6af8-4d1f-a40c-4e8b5a427776","remote_addr":"[fd00:1122:3344:101::1]:36766","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":348,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.533819039Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:35440"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.53395176Z","hostname":"buskin","pid":1566,"uri":"/sled-role","method":"GET","req_id":"47f17d25-fced-4db5-be45-fa14a7d3f5d1","remote_addr":"[fd00:1122:3344:101::1]:35440","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":48,"response_code":"200"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(2), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 59196224-69d2-4852-919f-d2f2c72a5aa0 (physical_disk), pool_id: 937da378-0aa4-4947-9249-421c34b23869 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 86b7c790-915b-4f21-96b4-02e8827a14a1 (physical_disk), pool_id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: a3252154-d160-4fdb-93b0-122104ade68c (physical_disk), pool_id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: aee9ce0b-9a21-48fc-92b4-7c7c335fb7ec (physical_disk), pool_id: 1cf41123-4d67-4150-b3da-35a0564ee66c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: bea0c832-d36f-49b5-b477-0ec8899bc64c (physical_disk), pool_id: fde4eef1-4144-45de-b52d-41ac67f3c852 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: e85d9967-76d1-4d22-9a79-fc11908f2bb1 (physical_disk), pool_id: 8c7eb352-05c8-4947-8210-6c96fcd2402e (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: e97f7703-8b3d-4383-b41c-d552128e153f (physical_disk), pool_id: 07bdd662-13a1-4ffd-b47f-4139ebd4bf64 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: ef3f4845-1340-4d2e-952c-17e7084624b2 (physical_disk), pool_id: 4057117e-b0fd-43cd-8023-6faae9ef843a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: f89e9763-ff52-4a9e-bbab-ce0c40311b76 (physical_disk), pool_id: e611b94c-e731-48e5-b021-59a1a0d163dd (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.564074191Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:385"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.564161582Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:35280"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.564564107Z","hostname":"buskin","pid":1566,"sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:895"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(2), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 59196224-69d2-4852-919f-d2f2c72a5aa0 (physical_disk), pool_id: 937da378-0aa4-4947-9249-421c34b23869 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 86b7c790-915b-4f21-96b4-02e8827a14a1 (physical_disk), pool_id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: a3252154-d160-4fdb-93b0-122104ade68c (physical_disk), pool_id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: aee9ce0b-9a21-48fc-92b4-7c7c335fb7ec (physical_disk), pool_id: 1cf41123-4d67-4150-b3da-35a0564ee66c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: bea0c832-d36f-49b5-b477-0ec8899bc64c (physical_disk), pool_id: fde4eef1-4144-45de-b52d-41ac67f3c852 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: e85d9967-76d1-4d22-9a79-fc11908f2bb1 (physical_disk), pool_id: 8c7eb352-05c8-4947-8210-6c96fcd2402e (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: e97f7703-8b3d-4383-b41c-d552128e153f (physical_disk), pool_id: 07bdd662-13a1-4ffd-b47f-4139ebd4bf64 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: ef3f4845-1340-4d2e-952c-17e7084624b2 (physical_disk), pool_id: 4057117e-b0fd-43cd-8023-6faae9ef843a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: f89e9763-ff52-4a9e-bbab-ce0c40311b76 (physical_disk), pool_id: e611b94c-e731-48e5-b021-59a1a0d163dd (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.564622497Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.564654628Z","hostname":"buskin","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/26a6a25d-ab9a-45d6-baed-89b7374cc198/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.564666768Z","hostname":"buskin","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.564677638Z","hostname":"buskin","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1298"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.564720728Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.564728908Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.564737258Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.564744318Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.70346981Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:44.703508221Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"1cf41123-4d67-4150-b3da-35a0564ee66c (zpool)"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:02:44.902598025Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:02:44.902710817Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.09883717Z","hostname":"buskin","pid":1566,"component":"VmmReservoirManager","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e20b6ac2-4792-45a8-adc6-0eb48d767a22)","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:02:45.100640169Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:238"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.158110332Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 1cf41123-4d67-4150-b3da-35a0564ee66c (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.166265296Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.166321817Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.166332847Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.166348907Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.166357817Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.166475599Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.166483419Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.200204847Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_1cf41123-4d67-4150-b3da-35a0564ee66c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.207582193Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_1cf41123-4d67-4150-b3da-35a0564ee66c","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.343272053Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.351121514Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.351191675Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.351203335Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.351213345Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.351220285Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.361063367Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.361102467Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"8c7eb352-05c8-4947-8210-6c96fcd2402e (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.457533983Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 8c7eb352-05c8-4947-8210-6c96fcd2402e (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.46599994Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.466060511Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.466074611Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.466085611Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.466097151Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.466195892Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.466205212Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.501436106Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_8c7eb352-05c8-4947-8210-6c96fcd2402e/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.510010054Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_8c7eb352-05c8-4947-8210-6c96fcd2402e","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.648989939Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.656756299Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.656808099Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.65681732Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.65682542Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.65683066Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.666925954Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.666960224Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"fde4eef1-4144-45de-b52d-41ac67f3c852 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.760895014Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: fde4eef1-4144-45de-b52d-41ac67f3c852 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.768547103Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.768616843Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.768629694Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.768640914Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.768649584Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.768768555Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.768779605Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.803530654Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_fde4eef1-4144-45de-b52d-41ac67f3c852/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.811489716Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_fde4eef1-4144-45de-b52d-41ac67f3c852","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.952577432Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.961677686Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.961708606Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.961721307Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.961728456Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.961750887Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.971668958Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:45.971702829Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.067500347Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.075838383Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.075879804Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.075891324Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.075921054Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.075933484Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.076045415Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.076053866Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e20b6ac2-4792-45a8-adc6-0eb48d767a22)","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:02:46.102917953Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.108613291Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.115574793Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.244509684Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.251380465Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.251411165Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.251417025Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.251421416Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.251426525Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.261659101Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.261683461Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"937da378-0aa4-4947-9249-421c34b23869 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.356737432Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.364586584Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.364614294Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.364631744Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.364688585Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.364699465Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.364822916Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.364829886Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.400410783Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.408700389Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_937da378-0aa4-4947-9249-421c34b23869","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.549983977Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.558649036Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.558687677Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.558739607Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.558749448Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.558756517Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.568633399Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.56866273Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.668753453Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.676620424Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.676660625Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.676669794Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.676700295Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.676711825Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.676815506Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.676826126Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.712910879Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.721107593Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.855749513Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.863540803Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.863581644Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.863589984Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.863596594Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.863601204Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.873503746Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.873528896Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"07bdd662-13a1-4ffd-b47f-4139ebd4bf64 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.968687078Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 07bdd662-13a1-4ffd-b47f-4139ebd4bf64 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.976559359Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.97659261Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.9765999Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.97663514Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.97664315Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.976804682Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:46.976814042Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.012715402Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_07bdd662-13a1-4ffd-b47f-4139ebd4bf64/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.021070849Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e20b6ac2-4792-45a8-adc6-0eb48d767a22)","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:02:47.104714662Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:238"} {"msg":"Looking for unencrypted datasets in oxp_07bdd662-13a1-4ffd-b47f-4139ebd4bf64","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.165758272Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.174216229Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.17427502Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.17428716Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.17429875Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.17430698Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.184448865Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.184475335Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"e611b94c-e731-48e5-b021-59a1a0d163dd (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.281947401Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: e611b94c-e731-48e5-b021-59a1a0d163dd (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.290614961Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.290651851Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.290658731Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.290701692Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.290716042Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.290910094Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.290918744Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.325298849Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_e611b94c-e731-48e5-b021-59a1a0d163dd/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.33315047Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_e611b94c-e731-48e5-b021-59a1a0d163dd","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.475735461Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.483439541Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.483475091Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.483505522Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.483514102Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.483519302Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.492542555Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.492576895Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"4057117e-b0fd-43cd-8023-6faae9ef843a (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.583769087Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 4057117e-b0fd-43cd-8023-6faae9ef843a (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.590604947Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.590639658Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.590648478Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.590670608Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.590682858Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.59086169Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.59087206Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.625878651Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_4057117e-b0fd-43cd-8023-6faae9ef843a/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.633103616Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_4057117e-b0fd-43cd-8023-6faae9ef843a","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.775805049Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.783115454Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.783155134Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.783212015Z","hostname":"buskin","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.783334226Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(2), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 26a6a25d-ab9a-45d6-baed-89b7374cc198 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 1cf41123-4d67-4150-b3da-35a0564ee66c (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 8c7eb352-05c8-4947-8210-6c96fcd2402e (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: fde4eef1-4144-45de-b52d-41ac67f3c852 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 07bdd662-13a1-4ffd-b47f-4139ebd4bf64 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: e611b94c-e731-48e5-b021-59a1a0d163dd (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4057117e-b0fd-43cd-8023-6faae9ef843a (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.783404087Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.783413857Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/26a6a25d-ab9a-45d6-baed-89b7374cc198/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.783483568Z","hostname":"buskin","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.78370587Z","hostname":"buskin","pid":1566,"sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:899"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.78371646Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.78372179Z","hostname":"buskin","pid":1566,"sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:925","generation":"Generation(2)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.78372774Z","hostname":"buskin","pid":1566,"sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:930"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.78373637Z","hostname":"buskin","pid":1566,"sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:935"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.783758841Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:123","gen":"Generation(2)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.783769551Z","hostname":"buskin","pid":1566,"sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:941"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.783806011Z","hostname":"buskin","pid":1566,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:799","gen":"Generation(2)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.783812941Z","hostname":"buskin","pid":1566,"sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:946"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.783909702Z","hostname":"buskin","pid":1566,"uri":"/omicron-physical-disks","method":"PUT","req_id":"b90dab8a-484a-412d-9575-9058d98c3ad1","remote_addr":"[fd00:1122:3344:101::1]:35280","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":3219554,"response_code":"200"} {"msg":"attempting to set up sled's datasets: DatasetsConfig { generation: Generation(1), datasets: {04bda49f-0e55-424b-bd21-ce70cffdd129 (dataset): DatasetConfig { id: 04bda49f-0e55-424b-bd21-ce70cffdd129 (dataset), name: DatasetName { pool_name: ZpoolName { id: 07bdd662-13a1-4ffd-b47f-4139ebd4bf64 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_6efc27f1-6196-413c-be8b-46c95f2b6949\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 050fd13b-394e-4bc6-a08b-36ccd6cadfb6 (dataset): DatasetConfig { id: 050fd13b-394e-4bc6-a08b-36ccd6cadfb6 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8c7eb352-05c8-4947-8210-6c96fcd2402e (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_89cce00e-2201-4929-a141-03bedac5eefe\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 07a5c272-a167-4d22-8959-afb737e84f16 (dataset): DatasetConfig { id: 07a5c272-a167-4d22-8959-afb737e84f16 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4057117e-b0fd-43cd-8023-6faae9ef843a (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 104ce690-a5ba-4c13-87f9-8608e7151b6c (dataset): DatasetConfig { id: 104ce690-a5ba-4c13-87f9-8608e7151b6c (dataset), name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 11db570a-634d-4a0c-9949-acbc3f94b13a (dataset): DatasetConfig { id: 11db570a-634d-4a0c-9949-acbc3f94b13a (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: TransientZone { name: \"oxz_clickhouse_a45c0062-6d07-46c4-8bfa-1d904fb44b6b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 17672084-f872-48a9-ba3e-294c220adbd5 (dataset): DatasetConfig { id: 17672084-f872-48a9-ba3e-294c220adbd5 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4057117e-b0fd-43cd-8023-6faae9ef843a (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_2c84a910-1bb9-4b4d-9f4f-e640af248108\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1b4f7b4c-c6bd-421a-8f53-dca08641e080 (dataset): DatasetConfig { id: 1b4f7b4c-c6bd-421a-8f53-dca08641e080 (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 20675205-6462-4240-bdf4-a53ea1bdc425 (dataset): DatasetConfig { id: 20675205-6462-4240-bdf4-a53ea1bdc425 (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_4692d59c-8317-4428-a479-0fd835976d71\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 276e9744-b1e4-4f27-b88c-4c939557bf1b (dataset): DatasetConfig { id: 276e9744-b1e4-4f27-b88c-4c939557bf1b (dataset), name: DatasetName { pool_name: ZpoolName { id: fde4eef1-4144-45de-b52d-41ac67f3c852 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2ac7316e-3ccb-4d25-ba80-3a40fae41bb2 (dataset): DatasetConfig { id: 2ac7316e-3ccb-4d25-ba80-3a40fae41bb2 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8c7eb352-05c8-4947-8210-6c96fcd2402e (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_1b508487-aacd-4ce6-b81c-e04080151870\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2ef6c753-735f-46b6-8eee-f264fe9cb506 (dataset): DatasetConfig { id: 2ef6c753-735f-46b6-8eee-f264fe9cb506 (dataset), name: DatasetName { pool_name: ZpoolName { id: fde4eef1-4144-45de-b52d-41ac67f3c852 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3426b96f-3093-4f87-9773-c5c63553e04f (dataset): DatasetConfig { id: 3426b96f-3093-4f87-9773-c5c63553e04f (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3c67a2ce-1d42-45f5-a4d7-7eb7b44bf7d6 (dataset): DatasetConfig { id: 3c67a2ce-1d42-45f5-a4d7-7eb7b44bf7d6 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3f6366b6-bf47-4c81-b674-7d7801f6c75d (dataset): DatasetConfig { id: 3f6366b6-bf47-4c81-b674-7d7801f6c75d (dataset), name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4a406b97-ca81-47f0-9527-52d618defeb7 (dataset): DatasetConfig { id: 4a406b97-ca81-47f0-9527-52d618defeb7 (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5254fa00-a5c8-4a51-984e-89b98bfa1511 (dataset): DatasetConfig { id: 5254fa00-a5c8-4a51-984e-89b98bfa1511 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4057117e-b0fd-43cd-8023-6faae9ef843a (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_55b5f278-a598-4dfc-9ed2-17d37f03db26\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6ad883b3-f69c-4332-bee2-d22db81956f4 (dataset): DatasetConfig { id: 6ad883b3-f69c-4332-bee2-d22db81956f4 (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_07abbe69-5f17-42ce-aeb3-7c56cc3fce3b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8164dcb1-40a0-4f76-8c75-94feb95f92bb (dataset): DatasetConfig { id: 8164dcb1-40a0-4f76-8c75-94feb95f92bb (dataset), name: DatasetName { pool_name: ZpoolName { id: 1cf41123-4d67-4150-b3da-35a0564ee66c (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 82e2ddba-973c-41a7-8a56-1cddece86d82 (dataset): DatasetConfig { id: 82e2ddba-973c-41a7-8a56-1cddece86d82 (dataset), name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_4715312e-6d68-4e37-b491-356caa08d118\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 87b7302b-d319-4c67-8a30-8f7d67dfec30 (dataset): DatasetConfig { id: 87b7302b-d319-4c67-8a30-8f7d67dfec30 (dataset), name: DatasetName { pool_name: ZpoolName { id: fde4eef1-4144-45de-b52d-41ac67f3c852 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_7b0cb0fc-c61a-4c0d-a609-1a73278738b2\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 88a12524-3523-4993-ba1e-189a6c50b8f8 (dataset): DatasetConfig { id: 88a12524-3523-4993-ba1e-189a6c50b8f8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8c7eb352-05c8-4947-8210-6c96fcd2402e (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 89b62b05-fc87-459b-9c73-616f8b7f51fe (dataset): DatasetConfig { id: 89b62b05-fc87-459b-9c73-616f8b7f51fe (dataset), name: DatasetName { pool_name: ZpoolName { id: 07bdd662-13a1-4ffd-b47f-4139ebd4bf64 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 8d5db858-d55d-45d7-abee-0740a59dd11d (dataset): DatasetConfig { id: 8d5db858-d55d-45d7-abee-0740a59dd11d (dataset), name: DatasetName { pool_name: ZpoolName { id: 07bdd662-13a1-4ffd-b47f-4139ebd4bf64 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_64cfc45a-47ac-4de4-a06d-03d924b7c0eb\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 91209603-872f-4feb-bfbe-2bba32ff04cb (dataset): DatasetConfig { id: 91209603-872f-4feb-bfbe-2bba32ff04cb (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 91677cf0-7c11-40c0-831f-cc3ee5eea879 (dataset): DatasetConfig { id: 91677cf0-7c11-40c0-831f-cc3ee5eea879 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1cf41123-4d67-4150-b3da-35a0564ee66c (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_978f682a-4055-4e9b-a120-004ade274154\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 92720901-fb38-49a7-9c74-39c78ce63c57 (dataset): DatasetConfig { id: 92720901-fb38-49a7-9c74-39c78ce63c57 (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9622bf7f-130e-4f64-b572-c71497beb9e4 (dataset): DatasetConfig { id: 9622bf7f-130e-4f64-b572-c71497beb9e4 (dataset), name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 96ad1651-3a5a-491d-b0a3-fb8df843c44d (dataset): DatasetConfig { id: 96ad1651-3a5a-491d-b0a3-fb8df843c44d (dataset), name: DatasetName { pool_name: ZpoolName { id: 1cf41123-4d67-4150-b3da-35a0564ee66c (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9d2e7d78-d5f2-4609-a8fb-0efee5ec1331 (dataset): DatasetConfig { id: 9d2e7d78-d5f2-4609-a8fb-0efee5ec1331 (dataset), name: DatasetName { pool_name: ZpoolName { id: fde4eef1-4144-45de-b52d-41ac67f3c852 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, a1ae62cd-2417-42e7-b9bc-96437fee558d (dataset): DatasetConfig { id: a1ae62cd-2417-42e7-b9bc-96437fee558d (dataset), name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_628eb80d-58d5-4fc4-ab6d-19758ef6d598\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a22b8134-8e76-4bf8-8dce-d53d035cb4b6 (dataset): DatasetConfig { id: a22b8134-8e76-4bf8-8dce-d53d035cb4b6 (dataset), name: DatasetName { pool_name: ZpoolName { id: e611b94c-e731-48e5-b021-59a1a0d163dd (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, a39fe806-34a6-4a95-bd8a-387f5f060f89 (dataset): DatasetConfig { id: a39fe806-34a6-4a95-bd8a-387f5f060f89 (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a420c3f7-d96d-4e03-bff6-87c860a68316 (dataset): DatasetConfig { id: a420c3f7-d96d-4e03-bff6-87c860a68316 (dataset), name: DatasetName { pool_name: ZpoolName { id: e611b94c-e731-48e5-b021-59a1a0d163dd (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_ccc17f21-b024-42d1-bbc4-b9b346bb75e8\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a4767a99-e2e0-4416-97b7-7af94161d8b3 (dataset): DatasetConfig { id: a4767a99-e2e0-4416-97b7-7af94161d8b3 (dataset), name: DatasetName { pool_name: ZpoolName { id: e611b94c-e731-48e5-b021-59a1a0d163dd (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_e61338ac-c2fc-4e0b-a5de-11c4906e111c\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a82b9b61-92c4-4d01-bd5d-561deb834747 (dataset): DatasetConfig { id: a82b9b61-92c4-4d01-bd5d-561deb834747 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4057117e-b0fd-43cd-8023-6faae9ef843a (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_15c1e91b-5078-4620-9a5f-350a017795ec\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, aa98f087-b710-4f6b-bc86-82b34d5f8c69 (dataset): DatasetConfig { id: aa98f087-b710-4f6b-bc86-82b34d5f8c69 (dataset), name: DatasetName { pool_name: ZpoolName { id: e611b94c-e731-48e5-b021-59a1a0d163dd (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ad022d76-8822-4a5f-b6bc-11a6954778be (dataset): DatasetConfig { id: ad022d76-8822-4a5f-b6bc-11a6954778be (dataset), name: DatasetName { pool_name: ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_622495e0-9798-45bd-b5d6-c46cbf9fcfcb\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ad0465dd-7b0f-4dc4-b9b0-a7c39b2010b4 (dataset): DatasetConfig { id: ad0465dd-7b0f-4dc4-b9b0-a7c39b2010b4 (dataset), name: DatasetName { pool_name: ZpoolName { id: 07bdd662-13a1-4ffd-b47f-4139ebd4bf64 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ade84c7e-da04-4138-9425-216477e6ccfc (dataset): DatasetConfig { id: ade84c7e-da04-4138-9425-216477e6ccfc (dataset), name: DatasetName { pool_name: ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, af83e4e4-5875-4a96-baaf-178269a32039 (dataset): DatasetConfig { id: af83e4e4-5875-4a96-baaf-178269a32039 (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: TransientZone { name: \"oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b2cec960-c615-417e-b236-347783e85092 (dataset): DatasetConfig { id: b2cec960-c615-417e-b236-347783e85092 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8c7eb352-05c8-4947-8210-6c96fcd2402e (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b9494fe1-959a-4951-8c45-d5dbdf2b4cea (dataset): DatasetConfig { id: b9494fe1-959a-4951-8c45-d5dbdf2b4cea (dataset), name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_5f24d564-5576-49da-8034-1d7c8e5fbf72\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, bcd845a9-dab4-47b1-8e62-31c48f04f787 (dataset): DatasetConfig { id: bcd845a9-dab4-47b1-8e62-31c48f04f787 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1cf41123-4d67-4150-b3da-35a0564ee66c (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_84a00632-3627-446e-b7bf-cd12db46e674\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c357f7b6-c2a7-4438-bcca-ffccfd1ea20a (dataset): DatasetConfig { id: c357f7b6-c2a7-4438-bcca-ffccfd1ea20a (dataset), name: DatasetName { pool_name: ZpoolName { id: 1cf41123-4d67-4150-b3da-35a0564ee66c (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c411bd38-6354-485c-99da-545357fa6784 (dataset): DatasetConfig { id: c411bd38-6354-485c-99da-545357fa6784 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_3e7c9535-3df2-415a-9579-4ac036ebade1\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c7b27118-ec1b-4785-8291-ca88c89b36d5 (dataset): DatasetConfig { id: c7b27118-ec1b-4785-8291-ca88c89b36d5 (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, c849d94a-d79b-40e4-8d35-70952e8e8095 (dataset): DatasetConfig { id: c849d94a-d79b-40e4-8d35-70952e8e8095 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1cf41123-4d67-4150-b3da-35a0564ee66c (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c8728b77-b4c7-456b-bcff-09873853533c (dataset): DatasetConfig { id: c8728b77-b4c7-456b-bcff-09873853533c (dataset), name: DatasetName { pool_name: ZpoolName { id: fde4eef1-4144-45de-b52d-41ac67f3c852 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_ead16108-e825-421b-96cb-00dff555d1f9\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c9aeea0e-3697-4c85-8d94-8db80681f228 (dataset): DatasetConfig { id: c9aeea0e-3697-4c85-8d94-8db80681f228 (dataset), name: DatasetName { pool_name: ZpoolName { id: e611b94c-e731-48e5-b021-59a1a0d163dd (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cc7da073-8a67-4b08-932b-7e60b71a44ed (dataset): DatasetConfig { id: cc7da073-8a67-4b08-932b-7e60b71a44ed (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_6d742fbc-b120-4fed-a908-3f3e097d6821\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ce3dbb58-2246-47d0-b6b0-b578a7a85eeb (dataset): DatasetConfig { id: ce3dbb58-2246-47d0-b6b0-b578a7a85eeb (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cebe9ffe-c9c5-4a41-a1d9-d0ccc48cef9e (dataset): DatasetConfig { id: cebe9ffe-c9c5-4a41-a1d9-d0ccc48cef9e (dataset), name: DatasetName { pool_name: ZpoolName { id: 07bdd662-13a1-4ffd-b47f-4139ebd4bf64 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d59a9525-732a-452c-b54d-a219efe9f868 (dataset): DatasetConfig { id: d59a9525-732a-452c-b54d-a219efe9f868 (dataset), name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d8c4804b-5563-43b3-a3be-a512bb1b885a (dataset): DatasetConfig { id: d8c4804b-5563-43b3-a3be-a512bb1b885a (dataset), name: DatasetName { pool_name: ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, da2b6a41-0bbd-4307-908e-446f53220243 (dataset): DatasetConfig { id: da2b6a41-0bbd-4307-908e-446f53220243 (dataset), name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ddc7acc9-4aa4-4c0b-b7eb-7269b62dfc15 (dataset): DatasetConfig { id: ddc7acc9-4aa4-4c0b-b7eb-7269b62dfc15 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4057117e-b0fd-43cd-8023-6faae9ef843a (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, df7ba82a-690b-434c-b055-55cbdb148d81 (dataset): DatasetConfig { id: df7ba82a-690b-434c-b055-55cbdb148d81 (dataset), name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_70112de1-b6b0-493a-8262-8524e63629bd\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e10eeea7-08f3-4669-a509-84db96df14e1 (dataset): DatasetConfig { id: e10eeea7-08f3-4669-a509-84db96df14e1 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e1d43af8-5cc4-4a81-9447-e92f5cc3a133 (dataset): DatasetConfig { id: e1d43af8-5cc4-4a81-9447-e92f5cc3a133 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8c7eb352-05c8-4947-8210-6c96fcd2402e (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, e502221d-6e4d-4752-b00d-f91a86923793 (dataset): DatasetConfig { id: e502221d-6e4d-4752-b00d-f91a86923793 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1cf41123-4d67-4150-b3da-35a0564ee66c (zpool), kind: External }, kind: TransientZone { name: \"oxz_oximeter_c68cd606-fc63-4bb0-b69c-9b8db711d413\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ef8c5fc5-a266-448a-9f18-f53b1aaaacc9 (dataset): DatasetConfig { id: ef8c5fc5-a266-448a-9f18-f53b1aaaacc9 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4057117e-b0fd-43cd-8023-6faae9ef843a (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, f033f6fd-6b3e-419b-93f9-5cb989138fb6 (dataset): DatasetConfig { id: f033f6fd-6b3e-419b-93f9-5cb989138fb6 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f6d8d0f2-95e3-4744-9822-b315c2997935 (dataset): DatasetConfig { id: f6d8d0f2-95e3-4744-9822-b315c2997935 (dataset), name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fc0aaee2-e5fd-4eb1-8c17-95d680e6159e (dataset): DatasetConfig { id: fc0aaee2-e5fd-4eb1-8c17-95d680e6159e (dataset), name: DatasetName { pool_name: ZpoolName { id: fde4eef1-4144-45de-b52d-41ac67f3c852 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fc9145f6-2456-4ae6-832b-c9ebf4fd8512 (dataset): DatasetConfig { id: fc9145f6-2456-4ae6-832b-c9ebf4fd8512 (dataset), name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fca39b02-158e-494e-ac23-a97fb3fad46d (dataset): DatasetConfig { id: fca39b02-158e-494e-ac23-a97fb3fad46d (dataset), name: DatasetName { pool_name: ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }} }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.815802201Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:460"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.815914132Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:40825"} {"msg":"datasets ensure","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.816410488Z","hostname":"buskin","pid":1566,"sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:861"} {"msg":"Received DatasetsEnsure { config: DatasetsConfig { generation: Generation(1), datasets: {04bda49f-0e55-424b-bd21-ce70cffdd129 (dataset): DatasetConfig { id: 04bda49f-0e55-424b-bd21-ce70cffdd129 (dataset), name: DatasetName { pool_name: ZpoolName { id: 07bdd662-13a1-4ffd-b47f-4139ebd4bf64 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_6efc27f1-6196-413c-be8b-46c95f2b6949\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 050fd13b-394e-4bc6-a08b-36ccd6cadfb6 (dataset): DatasetConfig { id: 050fd13b-394e-4bc6-a08b-36ccd6cadfb6 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8c7eb352-05c8-4947-8210-6c96fcd2402e (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_89cce00e-2201-4929-a141-03bedac5eefe\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 07a5c272-a167-4d22-8959-afb737e84f16 (dataset): DatasetConfig { id: 07a5c272-a167-4d22-8959-afb737e84f16 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4057117e-b0fd-43cd-8023-6faae9ef843a (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 104ce690-a5ba-4c13-87f9-8608e7151b6c (dataset): DatasetConfig { id: 104ce690-a5ba-4c13-87f9-8608e7151b6c (dataset), name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 11db570a-634d-4a0c-9949-acbc3f94b13a (dataset): DatasetConfig { id: 11db570a-634d-4a0c-9949-acbc3f94b13a (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: TransientZone { name: \"oxz_clickhouse_a45c0062-6d07-46c4-8bfa-1d904fb44b6b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 17672084-f872-48a9-ba3e-294c220adbd5 (dataset): DatasetConfig { id: 17672084-f872-48a9-ba3e-294c220adbd5 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4057117e-b0fd-43cd-8023-6faae9ef843a (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_2c84a910-1bb9-4b4d-9f4f-e640af248108\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1b4f7b4c-c6bd-421a-8f53-dca08641e080 (dataset): DatasetConfig { id: 1b4f7b4c-c6bd-421a-8f53-dca08641e080 (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 20675205-6462-4240-bdf4-a53ea1bdc425 (dataset): DatasetConfig { id: 20675205-6462-4240-bdf4-a53ea1bdc425 (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_4692d59c-8317-4428-a479-0fd835976d71\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 276e9744-b1e4-4f27-b88c-4c939557bf1b (dataset): DatasetConfig { id: 276e9744-b1e4-4f27-b88c-4c939557bf1b (dataset), name: DatasetName { pool_name: ZpoolName { id: fde4eef1-4144-45de-b52d-41ac67f3c852 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2ac7316e-3ccb-4d25-ba80-3a40fae41bb2 (dataset): DatasetConfig { id: 2ac7316e-3ccb-4d25-ba80-3a40fae41bb2 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8c7eb352-05c8-4947-8210-6c96fcd2402e (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_1b508487-aacd-4ce6-b81c-e04080151870\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2ef6c753-735f-46b6-8eee-f264fe9cb506 (dataset): DatasetConfig { id: 2ef6c753-735f-46b6-8eee-f264fe9cb506 (dataset), name: DatasetName { pool_name: ZpoolName { id: fde4eef1-4144-45de-b52d-41ac67f3c852 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3426b96f-3093-4f87-9773-c5c63553e04f (dataset): DatasetConfig { id: 3426b96f-3093-4f87-9773-c5c63553e04f (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3c67a2ce-1d42-45f5-a4d7-7eb7b44bf7d6 (dataset): DatasetConfig { id: 3c67a2ce-1d42-45f5-a4d7-7eb7b44bf7d6 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3f6366b6-bf47-4c81-b674-7d7801f6c75d (dataset): DatasetConfig { id: 3f6366b6-bf47-4c81-b674-7d7801f6c75d (dataset), name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4a406b97-ca81-47f0-9527-52d618defeb7 (dataset): DatasetConfig { id: 4a406b97-ca81-47f0-9527-52d618defeb7 (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5254fa00-a5c8-4a51-984e-89b98bfa1511 (dataset): DatasetConfig { id: 5254fa00-a5c8-4a51-984e-89b98bfa1511 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4057117e-b0fd-43cd-8023-6faae9ef843a (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_55b5f278-a598-4dfc-9ed2-17d37f03db26\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6ad883b3-f69c-4332-bee2-d22db81956f4 (dataset): DatasetConfig { id: 6ad883b3-f69c-4332-bee2-d22db81956f4 (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_07abbe69-5f17-42ce-aeb3-7c56cc3fce3b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8164dcb1-40a0-4f76-8c75-94feb95f92bb (dataset): DatasetConfig { id: 8164dcb1-40a0-4f76-8c75-94feb95f92bb (dataset), name: DatasetName { pool_name: ZpoolName { id: 1cf41123-4d67-4150-b3da-35a0564ee66c (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 82e2ddba-973c-41a7-8a56-1cddece86d82 (dataset): DatasetConfig { id: 82e2ddba-973c-41a7-8a56-1cddece86d82 (dataset), name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_4715312e-6d68-4e37-b491-356caa08d118\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 87b7302b-d319-4c67-8a30-8f7d67dfec30 (dataset): DatasetConfig { id: 87b7302b-d319-4c67-8a30-8f7d67dfec30 (dataset), name: DatasetName { pool_name: ZpoolName { id: fde4eef1-4144-45de-b52d-41ac67f3c852 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_7b0cb0fc-c61a-4c0d-a609-1a73278738b2\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 88a12524-3523-4993-ba1e-189a6c50b8f8 (dataset): DatasetConfig { id: 88a12524-3523-4993-ba1e-189a6c50b8f8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8c7eb352-05c8-4947-8210-6c96fcd2402e (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 89b62b05-fc87-459b-9c73-616f8b7f51fe (dataset): DatasetConfig { id: 89b62b05-fc87-459b-9c73-616f8b7f51fe (dataset), name: DatasetName { pool_name: ZpoolName { id: 07bdd662-13a1-4ffd-b47f-4139ebd4bf64 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 8d5db858-d55d-45d7-abee-0740a59dd11d (dataset): DatasetConfig { id: 8d5db858-d55d-45d7-abee-0740a59dd11d (dataset), name: DatasetName { pool_name: ZpoolName { id: 07bdd662-13a1-4ffd-b47f-4139ebd4bf64 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_64cfc45a-47ac-4de4-a06d-03d924b7c0eb\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 91209603-872f-4feb-bfbe-2bba32ff04cb (dataset): DatasetConfig { id: 91209603-872f-4feb-bfbe-2bba32ff04cb (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 91677cf0-7c11-40c0-831f-cc3ee5eea879 (dataset): DatasetConfig { id: 91677cf0-7c11-40c0-831f-cc3ee5eea879 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1cf41123-4d67-4150-b3da-35a0564ee66c (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_978f682a-4055-4e9b-a120-004ade274154\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 92720901-fb38-49a7-9c74-39c78ce63c57 (dataset): DatasetConfig { id: 92720901-fb38-49a7-9c74-39c78ce63c57 (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9622bf7f-130e-4f64-b572-c71497beb9e4 (dataset): DatasetConfig { id: 9622bf7f-130e-4f64-b572-c71497beb9e4 (dataset), name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 96ad1651-3a5a-491d-b0a3-fb8df843c44d (dataset): DatasetConfig { id: 96ad1651-3a5a-491d-b0a3-fb8df843c44d (dataset), name: DatasetName { pool_name: ZpoolName { id: 1cf41123-4d67-4150-b3da-35a0564ee66c (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9d2e7d78-d5f2-4609-a8fb-0efee5ec1331 (dataset): DatasetConfig { id: 9d2e7d78-d5f2-4609-a8fb-0efee5ec1331 (dataset), name: DatasetName { pool_name: ZpoolName { id: fde4eef1-4144-45de-b52d-41ac67f3c852 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, a1ae62cd-2417-42e7-b9bc-96437fee558d (dataset): DatasetConfig { id: a1ae62cd-2417-42e7-b9bc-96437fee558d (dataset), name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_628eb80d-58d5-4fc4-ab6d-19758ef6d598\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a22b8134-8e76-4bf8-8dce-d53d035cb4b6 (dataset): DatasetConfig { id: a22b8134-8e76-4bf8-8dce-d53d035cb4b6 (dataset), name: DatasetName { pool_name: ZpoolName { id: e611b94c-e731-48e5-b021-59a1a0d163dd (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, a39fe806-34a6-4a95-bd8a-387f5f060f89 (dataset): DatasetConfig { id: a39fe806-34a6-4a95-bd8a-387f5f060f89 (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a420c3f7-d96d-4e03-bff6-87c860a68316 (dataset): DatasetConfig { id: a420c3f7-d96d-4e03-bff6-87c860a68316 (dataset), name: DatasetName { pool_name: ZpoolName { id: e611b94c-e731-48e5-b021-59a1a0d163dd (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_ccc17f21-b024-42d1-bbc4-b9b346bb75e8\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a4767a99-e2e0-4416-97b7-7af94161d8b3 (dataset): DatasetConfig { id: a4767a99-e2e0-4416-97b7-7af94161d8b3 (dataset), name: DatasetName { pool_name: ZpoolName { id: e611b94c-e731-48e5-b021-59a1a0d163dd (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_e61338ac-c2fc-4e0b-a5de-11c4906e111c\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a82b9b61-92c4-4d01-bd5d-561deb834747 (dataset): DatasetConfig { id: a82b9b61-92c4-4d01-bd5d-561deb834747 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4057117e-b0fd-43cd-8023-6faae9ef843a (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_15c1e91b-5078-4620-9a5f-350a017795ec\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, aa98f087-b710-4f6b-bc86-82b34d5f8c69 (dataset): DatasetConfig { id: aa98f087-b710-4f6b-bc86-82b34d5f8c69 (dataset), name: DatasetName { pool_name: ZpoolName { id: e611b94c-e731-48e5-b021-59a1a0d163dd (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ad022d76-8822-4a5f-b6bc-11a6954778be (dataset): DatasetConfig { id: ad022d76-8822-4a5f-b6bc-11a6954778be (dataset), name: DatasetName { pool_name: ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_622495e0-9798-45bd-b5d6-c46cbf9fcfcb\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ad0465dd-7b0f-4dc4-b9b0-a7c39b2010b4 (dataset): DatasetConfig { id: ad0465dd-7b0f-4dc4-b9b0-a7c39b2010b4 (dataset), name: DatasetName { pool_name: ZpoolName { id: 07bdd662-13a1-4ffd-b47f-4139ebd4bf64 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ade84c7e-da04-4138-9425-216477e6ccfc (dataset): DatasetConfig { id: ade84c7e-da04-4138-9425-216477e6ccfc (dataset), name: DatasetName { pool_name: ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, af83e4e4-5875-4a96-baaf-178269a32039 (dataset): DatasetConfig { id: af83e4e4-5875-4a96-baaf-178269a32039 (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: TransientZone { name: \"oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b2cec960-c615-417e-b236-347783e85092 (dataset): DatasetConfig { id: b2cec960-c615-417e-b236-347783e85092 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8c7eb352-05c8-4947-8210-6c96fcd2402e (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b9494fe1-959a-4951-8c45-d5dbdf2b4cea (dataset): DatasetConfig { id: b9494fe1-959a-4951-8c45-d5dbdf2b4cea (dataset), name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_5f24d564-5576-49da-8034-1d7c8e5fbf72\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, bcd845a9-dab4-47b1-8e62-31c48f04f787 (dataset): DatasetConfig { id: bcd845a9-dab4-47b1-8e62-31c48f04f787 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1cf41123-4d67-4150-b3da-35a0564ee66c (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_84a00632-3627-446e-b7bf-cd12db46e674\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c357f7b6-c2a7-4438-bcca-ffccfd1ea20a (dataset): DatasetConfig { id: c357f7b6-c2a7-4438-bcca-ffccfd1ea20a (dataset), name: DatasetName { pool_name: ZpoolName { id: 1cf41123-4d67-4150-b3da-35a0564ee66c (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c411bd38-6354-485c-99da-545357fa6784 (dataset): DatasetConfig { id: c411bd38-6354-485c-99da-545357fa6784 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_3e7c9535-3df2-415a-9579-4ac036ebade1\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c7b27118-ec1b-4785-8291-ca88c89b36d5 (dataset): DatasetConfig { id: c7b27118-ec1b-4785-8291-ca88c89b36d5 (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, c849d94a-d79b-40e4-8d35-70952e8e8095 (dataset): DatasetConfig { id: c849d94a-d79b-40e4-8d35-70952e8e8095 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1cf41123-4d67-4150-b3da-35a0564ee66c (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c8728b77-b4c7-456b-bcff-09873853533c (dataset): DatasetConfig { id: c8728b77-b4c7-456b-bcff-09873853533c (dataset), name: DatasetName { pool_name: ZpoolName { id: fde4eef1-4144-45de-b52d-41ac67f3c852 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_ead16108-e825-421b-96cb-00dff555d1f9\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c9aeea0e-3697-4c85-8d94-8db80681f228 (dataset): DatasetConfig { id: c9aeea0e-3697-4c85-8d94-8db80681f228 (dataset), name: DatasetName { pool_name: ZpoolName { id: e611b94c-e731-48e5-b021-59a1a0d163dd (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cc7da073-8a67-4b08-932b-7e60b71a44ed (dataset): DatasetConfig { id: cc7da073-8a67-4b08-932b-7e60b71a44ed (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_6d742fbc-b120-4fed-a908-3f3e097d6821\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ce3dbb58-2246-47d0-b6b0-b578a7a85eeb (dataset): DatasetConfig { id: ce3dbb58-2246-47d0-b6b0-b578a7a85eeb (dataset), name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cebe9ffe-c9c5-4a41-a1d9-d0ccc48cef9e (dataset): DatasetConfig { id: cebe9ffe-c9c5-4a41-a1d9-d0ccc48cef9e (dataset), name: DatasetName { pool_name: ZpoolName { id: 07bdd662-13a1-4ffd-b47f-4139ebd4bf64 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d59a9525-732a-452c-b54d-a219efe9f868 (dataset): DatasetConfig { id: d59a9525-732a-452c-b54d-a219efe9f868 (dataset), name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d8c4804b-5563-43b3-a3be-a512bb1b885a (dataset): DatasetConfig { id: d8c4804b-5563-43b3-a3be-a512bb1b885a (dataset), name: DatasetName { pool_name: ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, da2b6a41-0bbd-4307-908e-446f53220243 (dataset): DatasetConfig { id: da2b6a41-0bbd-4307-908e-446f53220243 (dataset), name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ddc7acc9-4aa4-4c0b-b7eb-7269b62dfc15 (dataset): DatasetConfig { id: ddc7acc9-4aa4-4c0b-b7eb-7269b62dfc15 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4057117e-b0fd-43cd-8023-6faae9ef843a (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, df7ba82a-690b-434c-b055-55cbdb148d81 (dataset): DatasetConfig { id: df7ba82a-690b-434c-b055-55cbdb148d81 (dataset), name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_70112de1-b6b0-493a-8262-8524e63629bd\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e10eeea7-08f3-4669-a509-84db96df14e1 (dataset): DatasetConfig { id: e10eeea7-08f3-4669-a509-84db96df14e1 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e1d43af8-5cc4-4a81-9447-e92f5cc3a133 (dataset): DatasetConfig { id: e1d43af8-5cc4-4a81-9447-e92f5cc3a133 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8c7eb352-05c8-4947-8210-6c96fcd2402e (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, e502221d-6e4d-4752-b00d-f91a86923793 (dataset): DatasetConfig { id: e502221d-6e4d-4752-b00d-f91a86923793 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1cf41123-4d67-4150-b3da-35a0564ee66c (zpool), kind: External }, kind: TransientZone { name: \"oxz_oximeter_c68cd606-fc63-4bb0-b69c-9b8db711d413\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ef8c5fc5-a266-448a-9f18-f53b1aaaacc9 (dataset): DatasetConfig { id: ef8c5fc5-a266-448a-9f18-f53b1aaaacc9 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4057117e-b0fd-43cd-8023-6faae9ef843a (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, f033f6fd-6b3e-419b-93f9-5cb989138fb6 (dataset): DatasetConfig { id: f033f6fd-6b3e-419b-93f9-5cb989138fb6 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f6d8d0f2-95e3-4744-9822-b315c2997935 (dataset): DatasetConfig { id: f6d8d0f2-95e3-4744-9822-b315c2997935 (dataset), name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fc0aaee2-e5fd-4eb1-8c17-95d680e6159e (dataset): DatasetConfig { id: fc0aaee2-e5fd-4eb1-8c17-95d680e6159e (dataset), name: DatasetName { pool_name: ZpoolName { id: fde4eef1-4144-45de-b52d-41ac67f3c852 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fc9145f6-2456-4ae6-832b-c9ebf4fd8512 (dataset): DatasetConfig { id: fc9145f6-2456-4ae6-832b-c9ebf4fd8512 (dataset), name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fca39b02-158e-494e-ac23-a97fb3fad46d (dataset): DatasetConfig { id: fca39b02-158e-494e-ac23-a97fb3fad46d (dataset), name: DatasetName { pool_name: ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }} }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.81661055Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.81661635Z","hostname":"buskin","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/26a6a25d-ab9a-45d6-baed-89b7374cc198/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.81662067Z","hostname":"buskin","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested datasets', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.81662453Z","hostname":"buskin","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:909"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.831025359Z","hostname":"buskin","pid":1566,"name":"oxp_07bdd662-13a1-4ffd-b47f-4139ebd4bf64/crypt/zone/oxz_nexus_6efc27f1-6196-413c-be8b-46c95f2b6949","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.831073169Z","hostname":"buskin","pid":1566,"name":"oxp_07bdd662-13a1-4ffd-b47f-4139ebd4bf64/crypt/zone/oxz_nexus_6efc27f1-6196-413c-be8b-46c95f2b6949","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.831080209Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/07bdd662-13a1-4ffd-b47f-4139ebd4bf64/crypt/zone/oxz_nexus_6efc27f1-6196-413c-be8b-46c95f2b6949\"), full_name: \"oxp_07bdd662-13a1-4ffd-b47f-4139ebd4bf64/crypt/zone/oxz_nexus_6efc27f1-6196-413c-be8b-46c95f2b6949\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.887932676Z","hostname":"buskin","pid":1566,"name":"oxp_8c7eb352-05c8-4947-8210-6c96fcd2402e/crypt/zone/oxz_crucible_89cce00e-2201-4929-a141-03bedac5eefe","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.887968706Z","hostname":"buskin","pid":1566,"name":"oxp_8c7eb352-05c8-4947-8210-6c96fcd2402e/crypt/zone/oxz_crucible_89cce00e-2201-4929-a141-03bedac5eefe","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.887974136Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/8c7eb352-05c8-4947-8210-6c96fcd2402e/crypt/zone/oxz_crucible_89cce00e-2201-4929-a141-03bedac5eefe\"), full_name: \"oxp_8c7eb352-05c8-4947-8210-6c96fcd2402e/crypt/zone/oxz_crucible_89cce00e-2201-4929-a141-03bedac5eefe\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.944584771Z","hostname":"buskin","pid":1566,"name":"oxp_4057117e-b0fd-43cd-8023-6faae9ef843a/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.944619271Z","hostname":"buskin","pid":1566,"name":"oxp_4057117e-b0fd-43cd-8023-6faae9ef843a/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.944623711Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_4057117e-b0fd-43cd-8023-6faae9ef843a/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.988808096Z","hostname":"buskin","pid":1566,"name":"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.988845756Z","hostname":"buskin","pid":1566,"name":"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:47.988852036Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/external_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.034233155Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/zone/oxz_clickhouse_a45c0062-6d07-46c4-8bfa-1d904fb44b6b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.034269685Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/zone/oxz_clickhouse_a45c0062-6d07-46c4-8bfa-1d904fb44b6b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.034274635Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/937da378-0aa4-4947-9249-421c34b23869/crypt/zone/oxz_clickhouse_a45c0062-6d07-46c4-8bfa-1d904fb44b6b\"), full_name: \"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/zone/oxz_clickhouse_a45c0062-6d07-46c4-8bfa-1d904fb44b6b\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.088782428Z","hostname":"buskin","pid":1566,"name":"oxp_4057117e-b0fd-43cd-8023-6faae9ef843a/crypt/zone/oxz_nexus_2c84a910-1bb9-4b4d-9f4f-e640af248108","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.088817658Z","hostname":"buskin","pid":1566,"name":"oxp_4057117e-b0fd-43cd-8023-6faae9ef843a/crypt/zone/oxz_nexus_2c84a910-1bb9-4b4d-9f4f-e640af248108","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.088832928Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/4057117e-b0fd-43cd-8023-6faae9ef843a/crypt/zone/oxz_nexus_2c84a910-1bb9-4b4d-9f4f-e640af248108\"), full_name: \"oxp_4057117e-b0fd-43cd-8023-6faae9ef843a/crypt/zone/oxz_nexus_2c84a910-1bb9-4b4d-9f4f-e640af248108\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e20b6ac2-4792-45a8-adc6-0eb48d767a22)","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:02:48.106881815Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:238"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.144167299Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.14421358Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.14422124Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.192953853Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/zone/oxz_crucible_4692d59c-8317-4428-a479-0fd835976d71","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.192985503Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/zone/oxz_crucible_4692d59c-8317-4428-a479-0fd835976d71","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.192991653Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/937da378-0aa4-4947-9249-421c34b23869/crypt/zone/oxz_crucible_4692d59c-8317-4428-a479-0fd835976d71\"), full_name: \"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/zone/oxz_crucible_4692d59c-8317-4428-a479-0fd835976d71\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.248213623Z","hostname":"buskin","pid":1566,"name":"oxp_fde4eef1-4144-45de-b52d-41ac67f3c852/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.248253594Z","hostname":"buskin","pid":1566,"name":"oxp_fde4eef1-4144-45de-b52d-41ac67f3c852/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.248265584Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/fde4eef1-4144-45de-b52d-41ac67f3c852/crypt/zone\"), full_name: \"oxp_fde4eef1-4144-45de-b52d-41ac67f3c852/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.275143351Z","hostname":"buskin","pid":1566,"name":"oxp_8c7eb352-05c8-4947-8210-6c96fcd2402e/crypt/zone/oxz_crucible_pantry_1b508487-aacd-4ce6-b81c-e04080151870","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.275174192Z","hostname":"buskin","pid":1566,"name":"oxp_8c7eb352-05c8-4947-8210-6c96fcd2402e/crypt/zone/oxz_crucible_pantry_1b508487-aacd-4ce6-b81c-e04080151870","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.275179172Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/8c7eb352-05c8-4947-8210-6c96fcd2402e/crypt/zone/oxz_crucible_pantry_1b508487-aacd-4ce6-b81c-e04080151870\"), full_name: \"oxp_8c7eb352-05c8-4947-8210-6c96fcd2402e/crypt/zone/oxz_crucible_pantry_1b508487-aacd-4ce6-b81c-e04080151870\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.331604744Z","hostname":"buskin","pid":1566,"name":"oxp_fde4eef1-4144-45de-b52d-41ac67f3c852/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.331638304Z","hostname":"buskin","pid":1566,"name":"oxp_fde4eef1-4144-45de-b52d-41ac67f3c852/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.331643054Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_fde4eef1-4144-45de-b52d-41ac67f3c852/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.377066273Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.377097023Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.377102254Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_937da378-0aa4-4947-9249-421c34b23869/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.420789094Z","hostname":"buskin","pid":1566,"name":"oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/zone/oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.420831345Z","hostname":"buskin","pid":1566,"name":"oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/zone/oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.420836285Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/zone/oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db\"), full_name: \"oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/zone/oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.476412159Z","hostname":"buskin","pid":1566,"name":"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/zone/oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.476451159Z","hostname":"buskin","pid":1566,"name":"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/zone/oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.476456159Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/zone/oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2\"), full_name: \"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/zone/oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.531954402Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.531989762Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.531995702Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/937da378-0aa4-4947-9249-421c34b23869/crypt/zone\"), full_name: \"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.558586747Z","hostname":"buskin","pid":1566,"name":"oxp_4057117e-b0fd-43cd-8023-6faae9ef843a/crypt/zone/oxz_crucible_55b5f278-a598-4dfc-9ed2-17d37f03db26","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.558617407Z","hostname":"buskin","pid":1566,"name":"oxp_4057117e-b0fd-43cd-8023-6faae9ef843a/crypt/zone/oxz_crucible_55b5f278-a598-4dfc-9ed2-17d37f03db26","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.558627897Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/4057117e-b0fd-43cd-8023-6faae9ef843a/crypt/zone/oxz_crucible_55b5f278-a598-4dfc-9ed2-17d37f03db26\"), full_name: \"oxp_4057117e-b0fd-43cd-8023-6faae9ef843a/crypt/zone/oxz_crucible_55b5f278-a598-4dfc-9ed2-17d37f03db26\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.614834497Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/zone/oxz_cockroachdb_07abbe69-5f17-42ce-aeb3-7c56cc3fce3b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.614870988Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/zone/oxz_cockroachdb_07abbe69-5f17-42ce-aeb3-7c56cc3fce3b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.614876158Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/937da378-0aa4-4947-9249-421c34b23869/crypt/zone/oxz_cockroachdb_07abbe69-5f17-42ce-aeb3-7c56cc3fce3b\"), full_name: \"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/zone/oxz_cockroachdb_07abbe69-5f17-42ce-aeb3-7c56cc3fce3b\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.669440271Z","hostname":"buskin","pid":1566,"name":"oxp_1cf41123-4d67-4150-b3da-35a0564ee66c/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.669477131Z","hostname":"buskin","pid":1566,"name":"oxp_1cf41123-4d67-4150-b3da-35a0564ee66c/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.669482261Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1cf41123-4d67-4150-b3da-35a0564ee66c/crypt/debug\"), full_name: \"oxp_1cf41123-4d67-4150-b3da-35a0564ee66c/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.696177417Z","hostname":"buskin","pid":1566,"name":"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/zone/oxz_crucible_4715312e-6d68-4e37-b491-356caa08d118","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.696210297Z","hostname":"buskin","pid":1566,"name":"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/zone/oxz_crucible_4715312e-6d68-4e37-b491-356caa08d118","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.696215587Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/zone/oxz_crucible_4715312e-6d68-4e37-b491-356caa08d118\"), full_name: \"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/zone/oxz_crucible_4715312e-6d68-4e37-b491-356caa08d118\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.750300366Z","hostname":"buskin","pid":1566,"name":"oxp_fde4eef1-4144-45de-b52d-41ac67f3c852/crypt/zone/oxz_cockroachdb_7b0cb0fc-c61a-4c0d-a609-1a73278738b2","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.750327236Z","hostname":"buskin","pid":1566,"name":"oxp_fde4eef1-4144-45de-b52d-41ac67f3c852/crypt/zone/oxz_cockroachdb_7b0cb0fc-c61a-4c0d-a609-1a73278738b2","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.750331376Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/fde4eef1-4144-45de-b52d-41ac67f3c852/crypt/zone/oxz_cockroachdb_7b0cb0fc-c61a-4c0d-a609-1a73278738b2\"), full_name: \"oxp_fde4eef1-4144-45de-b52d-41ac67f3c852/crypt/zone/oxz_cockroachdb_7b0cb0fc-c61a-4c0d-a609-1a73278738b2\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.807028731Z","hostname":"buskin","pid":1566,"name":"oxp_8c7eb352-05c8-4947-8210-6c96fcd2402e/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.807057961Z","hostname":"buskin","pid":1566,"name":"oxp_8c7eb352-05c8-4947-8210-6c96fcd2402e/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.807062651Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_8c7eb352-05c8-4947-8210-6c96fcd2402e/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.851330878Z","hostname":"buskin","pid":1566,"name":"oxp_07bdd662-13a1-4ffd-b47f-4139ebd4bf64/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.851357849Z","hostname":"buskin","pid":1566,"name":"oxp_07bdd662-13a1-4ffd-b47f-4139ebd4bf64/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.851362778Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/07bdd662-13a1-4ffd-b47f-4139ebd4bf64/crypt/debug\"), full_name: \"oxp_07bdd662-13a1-4ffd-b47f-4139ebd4bf64/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.87858113Z","hostname":"buskin","pid":1566,"name":"oxp_07bdd662-13a1-4ffd-b47f-4139ebd4bf64/crypt/zone/oxz_crucible_64cfc45a-47ac-4de4-a06d-03d924b7c0eb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.87861745Z","hostname":"buskin","pid":1566,"name":"oxp_07bdd662-13a1-4ffd-b47f-4139ebd4bf64/crypt/zone/oxz_crucible_64cfc45a-47ac-4de4-a06d-03d924b7c0eb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.87862263Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/07bdd662-13a1-4ffd-b47f-4139ebd4bf64/crypt/zone/oxz_crucible_64cfc45a-47ac-4de4-a06d-03d924b7c0eb\"), full_name: \"oxp_07bdd662-13a1-4ffd-b47f-4139ebd4bf64/crypt/zone/oxz_crucible_64cfc45a-47ac-4de4-a06d-03d924b7c0eb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.93382495Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.9338535Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.93385824Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.978974795Z","hostname":"buskin","pid":1566,"name":"oxp_1cf41123-4d67-4150-b3da-35a0564ee66c/crypt/zone/oxz_crucible_978f682a-4055-4e9b-a120-004ade274154","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.978998286Z","hostname":"buskin","pid":1566,"name":"oxp_1cf41123-4d67-4150-b3da-35a0564ee66c/crypt/zone/oxz_crucible_978f682a-4055-4e9b-a120-004ade274154","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:48.979002646Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1cf41123-4d67-4150-b3da-35a0564ee66c/crypt/zone/oxz_crucible_978f682a-4055-4e9b-a120-004ade274154\"), full_name: \"oxp_1cf41123-4d67-4150-b3da-35a0564ee66c/crypt/zone/oxz_crucible_978f682a-4055-4e9b-a120-004ade274154\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.034458588Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/clickhouse","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.034489088Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/clickhouse","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.034494048Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/clickhouse\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.080316871Z","hostname":"buskin","pid":1566,"name":"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.080346822Z","hostname":"buskin","pid":1566,"name":"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.080351212Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/debug\"), full_name: \"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.106960876Z","hostname":"buskin","pid":1566,"name":"oxp_1cf41123-4d67-4150-b3da-35a0564ee66c/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.106984647Z","hostname":"buskin","pid":1566,"name":"oxp_1cf41123-4d67-4150-b3da-35a0564ee66c/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.106990447Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1cf41123-4d67-4150-b3da-35a0564ee66c/crypt/zone\"), full_name: \"oxp_1cf41123-4d67-4150-b3da-35a0564ee66c/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e20b6ac2-4792-45a8-adc6-0eb48d767a22)","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:02:49.110042928Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:238"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.134099176Z","hostname":"buskin","pid":1566,"name":"oxp_fde4eef1-4144-45de-b52d-41ac67f3c852/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.134130657Z","hostname":"buskin","pid":1566,"name":"oxp_fde4eef1-4144-45de-b52d-41ac67f3c852/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.134134457Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/fde4eef1-4144-45de-b52d-41ac67f3c852/crypt/debug\"), full_name: \"oxp_fde4eef1-4144-45de-b52d-41ac67f3c852/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.161312657Z","hostname":"buskin","pid":1566,"name":"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/zone/oxz_crucible_pantry_628eb80d-58d5-4fc4-ab6d-19758ef6d598","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.161344128Z","hostname":"buskin","pid":1566,"name":"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/zone/oxz_crucible_pantry_628eb80d-58d5-4fc4-ab6d-19758ef6d598","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.161349198Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/zone/oxz_crucible_pantry_628eb80d-58d5-4fc4-ab6d-19758ef6d598\"), full_name: \"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/zone/oxz_crucible_pantry_628eb80d-58d5-4fc4-ab6d-19758ef6d598\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.220250185Z","hostname":"buskin","pid":1566,"name":"oxp_e611b94c-e731-48e5-b021-59a1a0d163dd/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.220289896Z","hostname":"buskin","pid":1566,"name":"oxp_e611b94c-e731-48e5-b021-59a1a0d163dd/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.220296186Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/e611b94c-e731-48e5-b021-59a1a0d163dd/crypt/debug\"), full_name: \"oxp_e611b94c-e731-48e5-b021-59a1a0d163dd/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.2497838Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.249810951Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.249814911Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/external_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.295889566Z","hostname":"buskin","pid":1566,"name":"oxp_e611b94c-e731-48e5-b021-59a1a0d163dd/crypt/zone/oxz_crucible_pantry_ccc17f21-b024-42d1-bbc4-b9b346bb75e8","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.295919546Z","hostname":"buskin","pid":1566,"name":"oxp_e611b94c-e731-48e5-b021-59a1a0d163dd/crypt/zone/oxz_crucible_pantry_ccc17f21-b024-42d1-bbc4-b9b346bb75e8","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.295936137Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/e611b94c-e731-48e5-b021-59a1a0d163dd/crypt/zone/oxz_crucible_pantry_ccc17f21-b024-42d1-bbc4-b9b346bb75e8\"), full_name: \"oxp_e611b94c-e731-48e5-b021-59a1a0d163dd/crypt/zone/oxz_crucible_pantry_ccc17f21-b024-42d1-bbc4-b9b346bb75e8\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.351003275Z","hostname":"buskin","pid":1566,"name":"oxp_e611b94c-e731-48e5-b021-59a1a0d163dd/crypt/zone/oxz_crucible_e61338ac-c2fc-4e0b-a5de-11c4906e111c","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.351036506Z","hostname":"buskin","pid":1566,"name":"oxp_e611b94c-e731-48e5-b021-59a1a0d163dd/crypt/zone/oxz_crucible_e61338ac-c2fc-4e0b-a5de-11c4906e111c","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.351041615Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/e611b94c-e731-48e5-b021-59a1a0d163dd/crypt/zone/oxz_crucible_e61338ac-c2fc-4e0b-a5de-11c4906e111c\"), full_name: \"oxp_e611b94c-e731-48e5-b021-59a1a0d163dd/crypt/zone/oxz_crucible_e61338ac-c2fc-4e0b-a5de-11c4906e111c\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.408169985Z","hostname":"buskin","pid":1566,"name":"oxp_4057117e-b0fd-43cd-8023-6faae9ef843a/crypt/zone/oxz_nexus_15c1e91b-5078-4620-9a5f-350a017795ec","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.408207925Z","hostname":"buskin","pid":1566,"name":"oxp_4057117e-b0fd-43cd-8023-6faae9ef843a/crypt/zone/oxz_nexus_15c1e91b-5078-4620-9a5f-350a017795ec","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.408213305Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/4057117e-b0fd-43cd-8023-6faae9ef843a/crypt/zone/oxz_nexus_15c1e91b-5078-4620-9a5f-350a017795ec\"), full_name: \"oxp_4057117e-b0fd-43cd-8023-6faae9ef843a/crypt/zone/oxz_nexus_15c1e91b-5078-4620-9a5f-350a017795ec\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.469972033Z","hostname":"buskin","pid":1566,"name":"oxp_e611b94c-e731-48e5-b021-59a1a0d163dd/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.470015263Z","hostname":"buskin","pid":1566,"name":"oxp_e611b94c-e731-48e5-b021-59a1a0d163dd/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.470020074Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_e611b94c-e731-48e5-b021-59a1a0d163dd/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.518701556Z","hostname":"buskin","pid":1566,"name":"oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/zone/oxz_crucible_622495e0-9798-45bd-b5d6-c46cbf9fcfcb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.518730286Z","hostname":"buskin","pid":1566,"name":"oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/zone/oxz_crucible_622495e0-9798-45bd-b5d6-c46cbf9fcfcb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.518734236Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/zone/oxz_crucible_622495e0-9798-45bd-b5d6-c46cbf9fcfcb\"), full_name: \"oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/zone/oxz_crucible_622495e0-9798-45bd-b5d6-c46cbf9fcfcb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.579940038Z","hostname":"buskin","pid":1566,"name":"oxp_07bdd662-13a1-4ffd-b47f-4139ebd4bf64/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.579974838Z","hostname":"buskin","pid":1566,"name":"oxp_07bdd662-13a1-4ffd-b47f-4139ebd4bf64/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.579979828Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_07bdd662-13a1-4ffd-b47f-4139ebd4bf64/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.628447619Z","hostname":"buskin","pid":1566,"name":"oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.628480359Z","hostname":"buskin","pid":1566,"name":"oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.628485369Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.678232192Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/zone/oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.678263553Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/zone/oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.678280993Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/937da378-0aa4-4947-9249-421c34b23869/crypt/zone/oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1\"), full_name: \"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/zone/oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.738505954Z","hostname":"buskin","pid":1566,"name":"oxp_8c7eb352-05c8-4947-8210-6c96fcd2402e/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.738536915Z","hostname":"buskin","pid":1566,"name":"oxp_8c7eb352-05c8-4947-8210-6c96fcd2402e/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.738540855Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/8c7eb352-05c8-4947-8210-6c96fcd2402e/crypt/zone\"), full_name: \"oxp_8c7eb352-05c8-4947-8210-6c96fcd2402e/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.767300012Z","hostname":"buskin","pid":1566,"name":"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/zone/oxz_cockroachdb_5f24d564-5576-49da-8034-1d7c8e5fbf72","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.767327712Z","hostname":"buskin","pid":1566,"name":"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/zone/oxz_cockroachdb_5f24d564-5576-49da-8034-1d7c8e5fbf72","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.767331922Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/zone/oxz_cockroachdb_5f24d564-5576-49da-8034-1d7c8e5fbf72\"), full_name: \"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/zone/oxz_cockroachdb_5f24d564-5576-49da-8034-1d7c8e5fbf72\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.826889647Z","hostname":"buskin","pid":1566,"name":"oxp_1cf41123-4d67-4150-b3da-35a0564ee66c/crypt/zone/oxz_cockroachdb_84a00632-3627-446e-b7bf-cd12db46e674","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.826920927Z","hostname":"buskin","pid":1566,"name":"oxp_1cf41123-4d67-4150-b3da-35a0564ee66c/crypt/zone/oxz_cockroachdb_84a00632-3627-446e-b7bf-cd12db46e674","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.826924997Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1cf41123-4d67-4150-b3da-35a0564ee66c/crypt/zone/oxz_cockroachdb_84a00632-3627-446e-b7bf-cd12db46e674\"), full_name: \"oxp_1cf41123-4d67-4150-b3da-35a0564ee66c/crypt/zone/oxz_cockroachdb_84a00632-3627-446e-b7bf-cd12db46e674\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.89015522Z","hostname":"buskin","pid":1566,"name":"oxp_1cf41123-4d67-4150-b3da-35a0564ee66c/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.89018896Z","hostname":"buskin","pid":1566,"name":"oxp_1cf41123-4d67-4150-b3da-35a0564ee66c/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.89019376Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_1cf41123-4d67-4150-b3da-35a0564ee66c/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.938098375Z","hostname":"buskin","pid":1566,"name":"oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/zone/oxz_cockroachdb_3e7c9535-3df2-415a-9579-4ac036ebade1","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.938130175Z","hostname":"buskin","pid":1566,"name":"oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/zone/oxz_cockroachdb_3e7c9535-3df2-415a-9579-4ac036ebade1","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.938135415Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/zone/oxz_cockroachdb_3e7c9535-3df2-415a-9579-4ac036ebade1\"), full_name: \"oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/zone/oxz_cockroachdb_3e7c9535-3df2-415a-9579-4ac036ebade1\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.998586419Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.998627179Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:49.998632669Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/937da378-0aa4-4947-9249-421c34b23869/crypt/debug\"), full_name: \"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.02874313Z","hostname":"buskin","pid":1566,"name":"oxp_1cf41123-4d67-4150-b3da-35a0564ee66c/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.02879145Z","hostname":"buskin","pid":1566,"name":"oxp_1cf41123-4d67-4150-b3da-35a0564ee66c/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.02879667Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_1cf41123-4d67-4150-b3da-35a0564ee66c/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.078262871Z","hostname":"buskin","pid":1566,"name":"oxp_fde4eef1-4144-45de-b52d-41ac67f3c852/crypt/zone/oxz_crucible_ead16108-e825-421b-96cb-00dff555d1f9","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.078294121Z","hostname":"buskin","pid":1566,"name":"oxp_fde4eef1-4144-45de-b52d-41ac67f3c852/crypt/zone/oxz_crucible_ead16108-e825-421b-96cb-00dff555d1f9","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.078300192Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/fde4eef1-4144-45de-b52d-41ac67f3c852/crypt/zone/oxz_crucible_ead16108-e825-421b-96cb-00dff555d1f9\"), full_name: \"oxp_fde4eef1-4144-45de-b52d-41ac67f3c852/crypt/zone/oxz_crucible_ead16108-e825-421b-96cb-00dff555d1f9\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e20b6ac2-4792-45a8-adc6-0eb48d767a22)","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:02:50.111513174Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:238"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.141522554Z","hostname":"buskin","pid":1566,"name":"oxp_e611b94c-e731-48e5-b021-59a1a0d163dd/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.141557964Z","hostname":"buskin","pid":1566,"name":"oxp_e611b94c-e731-48e5-b021-59a1a0d163dd/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.141561954Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/e611b94c-e731-48e5-b021-59a1a0d163dd/crypt/zone\"), full_name: \"oxp_e611b94c-e731-48e5-b021-59a1a0d163dd/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.170739135Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/zone/oxz_external_dns_6d742fbc-b120-4fed-a908-3f3e097d6821","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.170774826Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/zone/oxz_external_dns_6d742fbc-b120-4fed-a908-3f3e097d6821","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.170784726Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/937da378-0aa4-4947-9249-421c34b23869/crypt/zone/oxz_external_dns_6d742fbc-b120-4fed-a908-3f3e097d6821\"), full_name: \"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/zone/oxz_external_dns_6d742fbc-b120-4fed-a908-3f3e097d6821\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.231409232Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/zone/oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.231444942Z","hostname":"buskin","pid":1566,"name":"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/zone/oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.231449502Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/937da378-0aa4-4947-9249-421c34b23869/crypt/zone/oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925\"), full_name: \"oxp_937da378-0aa4-4947-9249-421c34b23869/crypt/zone/oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.292883306Z","hostname":"buskin","pid":1566,"name":"oxp_07bdd662-13a1-4ffd-b47f-4139ebd4bf64/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.292921107Z","hostname":"buskin","pid":1566,"name":"oxp_07bdd662-13a1-4ffd-b47f-4139ebd4bf64/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.292926027Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/07bdd662-13a1-4ffd-b47f-4139ebd4bf64/crypt/zone\"), full_name: \"oxp_07bdd662-13a1-4ffd-b47f-4139ebd4bf64/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.323539243Z","hostname":"buskin","pid":1566,"name":"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.323572163Z","hostname":"buskin","pid":1566,"name":"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.323576853Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.375260226Z","hostname":"buskin","pid":1566,"name":"oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.375295237Z","hostname":"buskin","pid":1566,"name":"oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.375300267Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.424369893Z","hostname":"buskin","pid":1566,"name":"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.424404504Z","hostname":"buskin","pid":1566,"name":"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.424409594Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/zone\"), full_name: \"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.454528435Z","hostname":"buskin","pid":1566,"name":"oxp_4057117e-b0fd-43cd-8023-6faae9ef843a/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.454561675Z","hostname":"buskin","pid":1566,"name":"oxp_4057117e-b0fd-43cd-8023-6faae9ef843a/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.454566235Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/4057117e-b0fd-43cd-8023-6faae9ef843a/crypt/zone\"), full_name: \"oxp_4057117e-b0fd-43cd-8023-6faae9ef843a/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.484018999Z","hostname":"buskin","pid":1566,"name":"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/zone/oxz_external_dns_70112de1-b6b0-493a-8262-8524e63629bd","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.484054379Z","hostname":"buskin","pid":1566,"name":"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/zone/oxz_external_dns_70112de1-b6b0-493a-8262-8524e63629bd","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.48406588Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/zone/oxz_external_dns_70112de1-b6b0-493a-8262-8524e63629bd\"), full_name: \"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/zone/oxz_external_dns_70112de1-b6b0-493a-8262-8524e63629bd\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.542816156Z","hostname":"buskin","pid":1566,"name":"oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.542850896Z","hostname":"buskin","pid":1566,"name":"oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.542854936Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.592306297Z","hostname":"buskin","pid":1566,"name":"oxp_8c7eb352-05c8-4947-8210-6c96fcd2402e/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.592337147Z","hostname":"buskin","pid":1566,"name":"oxp_8c7eb352-05c8-4947-8210-6c96fcd2402e/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.592341107Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/8c7eb352-05c8-4947-8210-6c96fcd2402e/crypt/debug\"), full_name: \"oxp_8c7eb352-05c8-4947-8210-6c96fcd2402e/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.621620659Z","hostname":"buskin","pid":1566,"name":"oxp_1cf41123-4d67-4150-b3da-35a0564ee66c/crypt/zone/oxz_oximeter_c68cd606-fc63-4bb0-b69c-9b8db711d413","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.62165626Z","hostname":"buskin","pid":1566,"name":"oxp_1cf41123-4d67-4150-b3da-35a0564ee66c/crypt/zone/oxz_oximeter_c68cd606-fc63-4bb0-b69c-9b8db711d413","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.62166091Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1cf41123-4d67-4150-b3da-35a0564ee66c/crypt/zone/oxz_oximeter_c68cd606-fc63-4bb0-b69c-9b8db711d413\"), full_name: \"oxp_1cf41123-4d67-4150-b3da-35a0564ee66c/crypt/zone/oxz_oximeter_c68cd606-fc63-4bb0-b69c-9b8db711d413\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.684233046Z","hostname":"buskin","pid":1566,"name":"oxp_4057117e-b0fd-43cd-8023-6faae9ef843a/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.684267226Z","hostname":"buskin","pid":1566,"name":"oxp_4057117e-b0fd-43cd-8023-6faae9ef843a/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.684271576Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/4057117e-b0fd-43cd-8023-6faae9ef843a/crypt/debug\"), full_name: \"oxp_4057117e-b0fd-43cd-8023-6faae9ef843a/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.714598729Z","hostname":"buskin","pid":1566,"name":"oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.714632909Z","hostname":"buskin","pid":1566,"name":"oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.714637539Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/zone\"), full_name: \"oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.744686549Z","hostname":"buskin","pid":1566,"name":"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.7447214Z","hostname":"buskin","pid":1566,"name":"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.74472612Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.793693125Z","hostname":"buskin","pid":1566,"name":"oxp_fde4eef1-4144-45de-b52d-41ac67f3c852/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.793724776Z","hostname":"buskin","pid":1566,"name":"oxp_fde4eef1-4144-45de-b52d-41ac67f3c852/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.793729496Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_fde4eef1-4144-45de-b52d-41ac67f3c852/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.84263257Z","hostname":"buskin","pid":1566,"name":"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.842666711Z","hostname":"buskin","pid":1566,"name":"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.842671561Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_f63eb186-bef6-4979-a433-7a0f8d2a7f19/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.892403974Z","hostname":"buskin","pid":1566,"name":"oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.892441264Z","hostname":"buskin","pid":1566,"name":"oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.892445975Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/debug\"), full_name: \"oxp_8b3343df-fffa-4e0b-9470-981acc37b0ad/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Writing ledger to /pool/int/a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617/config/.omicron-datasets.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.921222192Z","hostname":"buskin","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/26a6a25d-ab9a-45d6-baed-89b7374cc198/config/.omicron-datasets.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.921539925Z","hostname":"buskin","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"datasets ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.921833828Z","hostname":"buskin","pid":1566,"sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:863"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.921979909Z","hostname":"buskin","pid":1566,"uri":"/datasets","method":"PUT","req_id":"bbfdbed5-3621-44bc-8877-bafe0658e9ef","remote_addr":"[fd00:1122:3344:101::1]:40825","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":3105949,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 5af17793-d4d8-49c8-b157-f36b23047925 (service), filesystem_pool: Some(ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bab050d7-a905-4fa6-ba9b-27247a4acab2 (service), filesystem_pool: Some(ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 190553b1-b763-4331-9460-7bafbd3e73db (service), filesystem_pool: Some(ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.95403033Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.954157681Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:39782"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.954555356Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.954566216Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.963808991Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.963855552Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.971903174Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.971946304Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.979766405Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.979808215Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/26a6a25d-ab9a-45d6-baed-89b7374cc198/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.979818645Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"ensure_all_omicron_zones: request reconciliation done","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.979886916Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:5013","num_zones_updated":0,"num_zones_to_be_added":3,"num_zones_to_be_removed":0} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:50.979894606Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:51.024615667Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:51.024663238Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:51.024667798Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:51.081469124Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:51.085561397Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:51.08979254Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:51.102188608Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:51.107019978Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:51.111600895Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e20b6ac2-4792-45a8-adc6-0eb48d767a22)","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:02:51.116684258Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:238"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:51.481323071Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"10.927533916s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/e20b6ac2-4792-45a8-adc6-0eb48d767a22)","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:02:52.118614809Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:238"} {"msg":"Profile for oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:52.734219632Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:52.734282983Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Booting oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2 zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:52.734508095Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:489"} {"msg":"Profile for oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:53.219922615Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:53.699097681Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:53.699167442Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2\") failed: Property not found. retry in 66.552029ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:53.706139934Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:53.706214635Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"Booting oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925 zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:53.706291685Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:489"} {"msg":"Booting oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:53.710761231Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:489"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2\") failed: Property not found. retry in 68.348065ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:53.783858816Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925\") failed: Property not found. retry in 69.201221ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:53.830675279Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db\") failed: Property not found. retry in 50.952952ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:53.836970664Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2\") failed: Property not found. retry in 292.720367ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:53.861142283Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db\") failed: Property not found. retry in 65.842067ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:53.897036664Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925\") failed: Property not found. retry in 77.305153ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:54.024160816Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:54.131605825Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4412"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:02:54.139124743Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"3612252707021847141"} {"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:54.139240364Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4422","gateway":"fd00:1122:3344:101::1"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:02:54.139456186Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14657506292094044072"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db\") failed: Property not found. retry in 237.164123ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:54.167337904Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Default route already exists","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:54.196206292Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4436","gateway":"fd00:1122:3344:101::1"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:54.196242502Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4501"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925\") failed: Property not found. retry in 171.610633ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:54.214095326Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2\") failed: Property not found. retry in 331.699171ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:54.264486286Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925\") failed: Property not found. retry in 597.932627ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:54.394998603Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db\") failed: Property not found. retry in 344.376372ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:54.406714144Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:54.482252924Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4542"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:54.482287404Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4569"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:54.530180979Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4582"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:54.530206349Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4453"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:54.589686203Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4495"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:54.589718423Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4549","rack_subnet":"fd00:1122:3344::"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2\") failed: Property not found. retry in 1.143595342s","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:54.591663963Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:54.612372847Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4560"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:54.612411747Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4633"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:54.721974898Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4688"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:54.722001019Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4694"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db\") failed: Property not found. retry in 761.831623ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:54.760084872Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:54.850944819Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4758"} {"msg":"configuring tfport service","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:54.85098248Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4585"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:02:54.904945647Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:02:54.905069068Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"refreshed tfport service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:54.951933561Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4618"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:54.951963652Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:370"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e20b6ac2-4792-45a8-adc6-0eb48d767a22)","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:54.984908262Z","hostname":"buskin","pid":1566,"component":"NexusNotifierTask","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/nexus.rs:530"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:54.985504518Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:420"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925\") failed: Property not found. retry in 903.015153ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:55.007802368Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.033583084Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:717"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.033741846Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:762"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.034352382Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:765","uptime":"2"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.034364062Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:444","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: Some(\n None,\n ),\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n tx_eq: None,\n },\n },\n },\n}"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.094605934Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4239"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.152162638Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.152223869Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.152236439Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.152243249Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.152249659Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.152255809Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.152263169Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.152269239Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.152293719Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.152300209Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.1523069Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.15231456Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.15232113Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.1523255Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.15232997Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.15233434Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.15233804Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.15234202Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.15234612Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.15235027Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.1523583Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.1523625Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:02:55.15236652Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db\") failed: Property not found. retry in 1.039289087s","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:55.533568304Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2\") failed: Property not found. retry in 1.113201912s","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:55.746477802Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925\") failed: Property not found. retry in 524.056217ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:55.920372727Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925\") failed: Property not found. retry in 1.390884125s","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:56.453971364Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db\") failed: Property not found. retry in 675.765346ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:56.582077126Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2\") failed: Property not found. retry in 593.875672ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:56.867541832Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db\") failed: Property not found. retry in 513.019492ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:57.266168786Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2\") failed: Property not found. retry in 618.063766ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:57.469631816Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db\") failed: Property not found. retry in 505.009616ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:57.789136703Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925\") failed: Property not found. retry in 748.908038ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:57.853591699Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2\") failed: Property not found. retry in 957.217135ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:58.09695535Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db\") failed: Property not found. retry in 789.175207ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:58.302824225Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925\") failed: Property not found. retry in 1.39400057s","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:58.61237316Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e20b6ac2-4792-45a8-adc6-0eb48d767a22)","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:02:59.9873823Z","hostname":"buskin","pid":1566,"component":"NexusNotifierTask","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"sled-agent/src/nexus.rs:530"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.028223682Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3426","zone":"oxz_internal_dns_5af17793-d4d8-49c8-b157-f36b23047925"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.028264802Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3426","zone":"oxz_internal_dns_190553b1-b763-4331-9460-7bafbd3e73db"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.028272772Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3426","zone":"oxz_internal_dns_bab050d7-a905-4fa6-ba9b-27247a4acab2"} {"msg":"Writing ledger to /pool/int/a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.028331643Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/26a6a25d-ab9a-45d6-baed-89b7374cc198/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.028648836Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.02897921Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"638f806f-d8b6-49b6-b99f-7395b4a28247","remote_addr":"[fd00:1122:3344:101::1]:39782","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":9074654,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.031049121Z","hostname":"buskin","pid":1566,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:658"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.07750106Z","hostname":"buskin","pid":1566,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:658"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.130025652Z","hostname":"buskin","pid":1566,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:658"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.176598953Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:694"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.176611033Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.208686294Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.209469502Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.209474692Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 5af17793-d4d8-49c8-b157-f36b23047925 (service), filesystem_pool: Some(ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: bab050d7-a905-4fa6-ba9b-27247a4acab2 (service), filesystem_pool: Some(ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 190553b1-b763-4331-9460-7bafbd3e73db (service), filesystem_pool: Some(ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 02996ddd-87ce-45fb-9c94-4c7ed57603e1 (service), filesystem_pool: Some(ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 3cf1181d-5bb2-4c43-b90d-c1af1a0b0f48, kind: Service { id: 02996ddd-87ce-45fb-9c94-4c7ed57603e1 }, name: Name(\"ntp-02996ddd-87ce-45fb-9c94-4c7ed57603e1\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 17])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.238680774Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.238779905Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:44302"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.238995097Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.239002737Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 937da378-0aa4-4947-9249-421c34b23869 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.250507016Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.250559476Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: f63eb186-bef6-4979-a433-7a0f8d2a7f19 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.260184156Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.260221966Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b3343df-fffa-4e0b-9470-981acc37b0ad (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.269640963Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.269694864Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/26a6a25d-ab9a-45d6-baed-89b7374cc198/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.269873546Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"ensure_all_omicron_zones: request reconciliation done","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.269992247Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:5013","num_zones_updated":0,"num_zones_to_be_added":1,"num_zones_to_be_removed":0} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.270003147Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.281082181Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.312782349Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.313385475Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.313390215Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.3429566Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 180, 17]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.34296398Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1321","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.343326704Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.343334924Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.343579546Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.34396389Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.367782666Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.386469629Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.999835669Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1 zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:00.9999748Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:489"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1\") failed: Property not found. retry in 29.532527ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:01.147040238Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1\") failed: Property not found. retry in 51.393951ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:01.184857888Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1\") failed: Property not found. retry in 210.256254ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:01.248557906Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1\") failed: Property not found. retry in 561.627873ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:01.471639388Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1\") failed: Property not found. retry in 770.885789ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:02.043804894Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1\") failed: Property not found. retry in 1.444394123s","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:02.825210809Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:04.139899287Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"3612252707021847141"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:04.141030299Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14657506292094044072"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1\") failed: Property not found. retry in 840.736558ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:04.279931483Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:04.906990755Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:04.907106586Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1\") failed: Property not found. retry in 631.107029ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:05.129345919Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1\") failed: Property not found. retry in 533.830338ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:05.769970671Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:06.325809357Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3426","zone":"oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1"} {"msg":"Writing ledger to /pool/int/a04dc0bd-5e2e-4b17-b6b1-15a3a45a8617/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:06.325866138Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/26a6a25d-ab9a-45d6-baed-89b7374cc198/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:06.326231972Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:06.326574845Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"4ff75bb7-b82c-470f-b4c5-9af37c3b3001","remote_addr":"[fd00:1122:3344:101::1]:44302","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":6087729,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:06.328758428Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:735"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:06.358734787Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:06.358906229Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:46318"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:06.368437007Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4020"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:06.368550648Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"f7fdecac-ed86-46bc-89e1-2197831bd525","remote_addr":"[fd00:1122:3344:101::1]:46318","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:855","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9534,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:06.370882883Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:06.548339144Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:06.548493686Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:63896"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:09.080245955Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13845490965388833884"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:09.113536789Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12279443132116500813"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:10.032049509Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11598486620168652696"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:10.557632934Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4020"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:10.557770535Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"a6845795-63d0-4cbc-bb76-c623664555a4","remote_addr":"[fd00:1122:3344:101::1]:63896","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:855","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009220,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:10.560510964Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:10.690328713Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:10.690518025Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:42321"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:10.699141174Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4020"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:10.699221535Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"bbf7005f-6222-4b1e-b453-9ce5bd9015a5","remote_addr":"[fd00:1122:3344:101::1]:42321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:855","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8648,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:10.700717571Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:10.929383461Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:10.929571923Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:39733"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:10.938335543Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4020"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:10.938432734Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"af538cca-8376-4173-92cc-f025eadf5139","remote_addr":"[fd00:1122:3344:101::1]:39733","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:855","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8780,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:10.94094216Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:11.15226116Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:11.152422172Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:50898"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:14.142393461Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"3612252707021847141"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:14.142503372Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14657506292094044072"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:14.908979103Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:14.909101044Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:15.161292497Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"48d8bcf3-f3fb-4961-9b16-0d6e1e857841","remote_addr":"[fd00:1122:3344:101::1]:50898","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":4008818,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:15.164815664Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:15.164828973Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:15.421528703Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:15.421708755Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:40299"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:15.430785658Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"dc481a63-d241-4be4-96a0-8fdbec936c14","remote_addr":"[fd00:1122:3344:101::1]:40299","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":9005,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:15.434605448Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:15.434613208Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:15.653252295Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:15.653420336Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:52992"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:15.661978125Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"9d45a9c0-6661-4b5d-bbae-18de771d06ad","remote_addr":"[fd00:1122:3344:101::1]:52992","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":8462,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:15.665923885Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:15.665934136Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:16.061502517Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:16.061620999Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:43716"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:16.071033996Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"cdc3c74d-4772-40f2-af92-e9a85885f472","remote_addr":"[fd00:1122:3344:101::1]:43716","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":9268,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:16.073282539Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:16.073290099Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:16.329846107Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"688312343100055529"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:16.329965958Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2196739395682377057"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:16.638593143Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:16.638782305Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:58882"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:16.647742188Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"b49c7dbd-3e5f-4820-b800-232392e7f7d1","remote_addr":"[fd00:1122:3344:101::1]:58882","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":8871,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:16.64984413Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:16.64985475Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:17.26125462Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:17.261407261Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:65242"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:17.269658216Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"eb5056e9-7958-4f47-b639-f6d1f3f9963e","remote_addr":"[fd00:1122:3344:101::1]:65242","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":8184,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:17.271998301Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:17.272006311Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:17.447365651Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"338.399594ms"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:17.86098661Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:17.861143441Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:35599"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:17.869534268Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"ff28f70f-b5c6-48c9-aee2-8c0379a65ca2","remote_addr":"[fd00:1122:3344:101::1]:35599","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":8312,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:17.8716509Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:17.8716683Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:18.630640023Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:18.630819015Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:60747"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:18.639877499Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"709de2fe-c8b5-45e5-9be7-552b4afba941","remote_addr":"[fd00:1122:3344:101::1]:60747","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":8964,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:18.64191745Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:18.64192437Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:19.103137329Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:19.10328716Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:51505"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:19.107619505Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13845490965388833884"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:19.11193876Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"8e5cecf5-e210-4806-bcc5-314a30be4b8d","remote_addr":"[fd00:1122:3344:101::1]:51505","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":8474,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:19.114085302Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:19.114092882Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:19.11489835Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12279443132116500813"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:20.033487641Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11598486620168652696"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:20.284950476Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:20.285137758Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:42453"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:20.293956749Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"4a4a309a-a32e-4005-bf85-dad253d633d3","remote_addr":"[fd00:1122:3344:101::1]:42453","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":8729,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:20.29590594Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:20.29591391Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:21.379514913Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:21.379697655Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:53915"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:21.387093262Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"000b6863-354e-435c-a441-55fcaf5c4b18","remote_addr":"[fd00:1122:3344:101::1]:53915","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":7321,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:21.390156113Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:21.390165574Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:22.200696659Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:22.20087593Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:63968"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:22.208040424Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"a42ae022-a3bf-4cdb-b845-9418b7beec9c","remote_addr":"[fd00:1122:3344:101::1]:63968","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":7099,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:22.209826083Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:22.209830973Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:23.835075296Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:23.835200958Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:42413"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:23.843673555Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"74df7b26-3df6-45ce-9d3d-11214241f8a8","remote_addr":"[fd00:1122:3344:101::1]:42413","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":8349,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:23.845505504Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:23.845511424Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:24.145050055Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"3612252707021847141"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:24.145170447Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14657506292094044072"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:24.481013043Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41159"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:24.481245635Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"143ddc9d-3105-4ac3-bbfc-2bc98c7ec270","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41159","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":100,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:24.91095116Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:24.911065372Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:25.896191289Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:25.896366421Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:51754"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:25.905102941Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"19be7928-bb73-41d5-bc47-07446eb0772f","remote_addr":"[fd00:1122:3344:101::1]:51754","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":8658,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:25.906787978Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:25.906792069Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:26.332550482Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"688312343100055529"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:26.332671513Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2196739395682377057"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:28.084323251Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:28.084500313Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:51681"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:28.093672757Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"901193a4-44c9-4b04-b514-987151d0bee5","remote_addr":"[fd00:1122:3344:101::1]:51681","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":9087,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:28.096979332Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:28.096986341Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:29.109534742Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13845490965388833884"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:29.115803176Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12279443132116500813"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:30.035236256Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11598486620168652696"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:31.398752149Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:31.398932301Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:53640"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:31.407635671Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"2432a59e-78e5-4731-9b27-3b3d9fd97314","remote_addr":"[fd00:1122:3344:101::1]:53640","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":8634,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:31.410902654Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:31.410911224Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:32.820967297Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"717.718128ms"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:34.146844481Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"3612252707021847141"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:34.147106214Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14657506292094044072"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:34.912578274Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:34.912691145Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:35.612315816Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:35.612443558Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:41101"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:35.621138327Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"bf7efb9c-2e6f-4548-babb-c3eea1eff2d6","remote_addr":"[fd00:1122:3344:101::1]:41101","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":8507,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:35.623247819Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:35.623268159Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:36.334852043Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"688312343100055529"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:36.334978385Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2196739395682377057"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:38.554599213Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:38.554756245Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:57831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:38.563107381Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"84bf74db-3ba1-4d66-92ff-8681217eb2e1","remote_addr":"[fd00:1122:3344:101::1]:57831","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":8260,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:38.565514366Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:38.565521696Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:39.111651732Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13845490965388833884"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:39.116932176Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12279443132116500813"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:40.037367076Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11598486620168652696"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:44.148650488Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"3612252707021847141"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:44.149148123Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"14657506292094044072"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:44.91429228Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:44.914409011Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:45.069919386Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:45.070105698Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:39096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:45.079388454Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"09dd734d-7f2d-451b-931b-1cf36797505d","remote_addr":"[fd00:1122:3344:101::1]:39096","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":9200,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:45.081850179Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:45.081855339Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:46.3366376Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"688312343100055529"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:46.336755811Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2196739395682377057"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:48.575576108Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"527.074398ms"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:49.135607568Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13845490965388833884"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:49.135727549Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12279443132116500813"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:49.717777817Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:49.717940168Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:53403"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:49.726649358Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"4ec18b4c-ad38-4350-8c6c-2b4889604768","remote_addr":"[fd00:1122:3344:101::1]:53403","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":8606,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:49.73065601Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:49.73066306Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:50.040251935Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11598486620168652696"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:54.515863747Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47117"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:54.515945227Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"8e597492-4353-4ada-b03d-bd3648714fcc","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47117","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":34,"response_code":"200"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:54.744385945Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:54.744582777Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:42349"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:54.753032885Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"0b606444-d514-4055-8b7e-8bcc24b1251e","remote_addr":"[fd00:1122:3344:101::1]:42349","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":8346,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:03:54.755715802Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:03:54.755723402Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:54.916892066Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:54.917017897Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:56.338554438Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"688312343100055529"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:56.338833121Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2196739395682377057"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:59.137479815Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"13845490965388833884"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:03:59.137787618Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"12279443132116500813"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:04:00.042532705Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"11598486620168652696"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:04:03.745658585Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:04:03.745842837Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:42300"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:04:03.754971561Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"e136a8b2-6281-4ec9-8174-6d9a25843aa1","remote_addr":"[fd00:1122:3344:101::1]:42300","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":9027,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:04:03.757344846Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:04:03.757352366Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:04:04.138318738Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"2.924567329s"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:04:04.919275578Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:04:04.91939633Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:04:06.340851969Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"688312343100055529"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:04:06.341254544Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"2196739395682377057"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:04:14.921817732Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:04:14.921948083Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:04:15.271991726Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:04:15.272203008Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:50425"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:04:15.281216481Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"04f2a56c-be2e-4c80-bc56-2aa8f4bff002","remote_addr":"[fd00:1122:3344:101::1]:50425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":8921,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:04:15.284127731Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:04:15.284133291Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:04:22.099002245Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"5.441935937s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:04:24.547808579Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38739"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:04:24.54788699Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"536b82a6-58fe-4aca-af9a-f94a9aa2336c","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38739","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":30,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:04:24.923671989Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:04:24.92378776Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:04:31.552129619Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:04:31.552292061Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:46237"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:04:31.561750388Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"2ac0b771-52cd-4ade-a121-78cb2f49d272","remote_addr":"[fd00:1122:3344:101::1]:46237","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":9371,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:04:31.564425906Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:04:31.564434136Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:04:34.925743766Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:04:34.925855398Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:04:42.575508649Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"11.949905666s"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:04:44.92715021Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:04:44.927264651Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:04:45.096314145Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:04:45.096497227Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:45249"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:04:45.106239788Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"48a8006c-7111-424c-b109-78e35dea4c04","remote_addr":"[fd00:1122:3344:101::1]:45249","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":9643,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:04:45.108929416Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:04:45.108934646Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:04:54.581239518Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51356"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:04:54.581326649Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"438f3c3f-5100-4efd-9c29-544b2bd46f42","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51356","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":36,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:04:54.92919631Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:04:54.929322131Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:05:04.931549082Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:05:04.931660903Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:05:05.454855143Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:05:05.455032025Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:48538"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:05:05.464962297Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"44aa53b4-84ec-4ba2-81bc-5d2cbaface1c","remote_addr":"[fd00:1122:3344:101::1]:48538","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":9840,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:05:05.467725966Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:05:05.467736366Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:05:09.56188788Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"10.313044852s"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:05:14.93456054Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:05:14.934672591Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:05:24.614113121Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50747"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:05:24.614196742Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"678a34c3-a7dc-4349-88c0-eb3413ea30ba","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50747","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":30,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:05:24.936400467Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:05:24.936515129Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:05:26.487733528Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:05:26.48794129Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:45664"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:05:26.496885873Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"54a2347e-5b61-42b6-9ec2-dca78ae943a6","remote_addr":"[fd00:1122:3344:101::1]:45664","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":8838,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:05:26.499653641Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:05:26.499659411Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:05:34.909991602Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"24.918075707s"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:05:34.938983921Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:05:34.939105502Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:05:44.940737567Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:05:44.94100185Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:05:54.201450895Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:05:54.201680837Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:58367"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:05:54.21070798Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"4fa40dcd-d668-471f-a481-3022f1f774b6","remote_addr":"[fd00:1122:3344:101::1]:58367","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":8922,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:05:54.21356817Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:05:54.21357373Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:05:54.647565909Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40459"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:05:54.64769136Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"f324b691-49e9-4121-93ed-054fedea81d2","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40459","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":42,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:05:54.942596114Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:05:54.942716396Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:06:04.944771813Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:06:04.944874244Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:06:13.437839047Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:06:13.437960419Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:41440"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:06:13.447186034Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"63068cac-7960-4007-81c0-a7614ecbd317","remote_addr":"[fd00:1122:3344:101::1]:41440","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":9054,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:06:13.449895422Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:06:13.449905032Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:06:14.863126957Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"86.776582386s"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:06:14.946552828Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:06:14.946669349Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:06:24.68054603Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44843"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:06:24.680660592Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"0ba4a1a4-383b-4381-ab9a-39b67021fea3","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44843","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":41,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:06:24.948720308Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:06:24.948956991Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:06:34.950530823Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:06:34.950765486Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:06:44.952324171Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:06:44.952423252Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:06:54.715082929Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58960"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:06:54.71519888Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"4184a191-0cdb-475b-b76e-bd78b9a68964","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58960","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":44,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:06:54.955366539Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:06:54.955573131Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:07:00.532417468Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:07:00.532578999Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:56718"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:07:00.541587872Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"0d272e2f-ad42-4a5d-8f49-48062c4f6d37","remote_addr":"[fd00:1122:3344:101::1]:56718","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":8933,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:07:00.54428076Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:07:00.54428764Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:07:04.958735611Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:07:04.958844842Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:07:14.960540535Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:07:14.960818007Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:07:24.747926098Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36171"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:07:24.748043269Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"baee57a1-2e21-4b24-8157-9a4ae79fe6c9","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36171","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":50,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:07:24.962871426Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:07:24.963130949Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:07:34.964823825Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:07:34.965024247Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:07:44.967119935Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:07:44.967381918Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:07:47.785121069Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:07:54.781732008Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65016"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:07:54.781818839Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"5727bfa8-ee25-4a58-b441-10cef5510a0a","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65016","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":39,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:07:54.968479565Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:07:54.968602467Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:07:56.126060483Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:07:56.126274945Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:37704"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:07:56.135188967Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"d74baa96-5ea6-4730-902f-26cb918bc113","remote_addr":"[fd00:1122:3344:101::1]:37704","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":8837,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:07:56.137711313Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:07:56.137716963Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:07:56.6751587Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"157.95956569s"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:08:04.971378304Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:08:04.971480335Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:08:14.973876888Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:08:14.97412933Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:08:24.815183269Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42669"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:08:24.81529636Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"e5599b51-6eab-48bb-bddd-05cb78cf71a7","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42669","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":30,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:08:24.975878977Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:08:24.976015649Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:08:34.978024047Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:08:34.978229149Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:08:44.97952393Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:08:44.979783953Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:08:52.39361337Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:08:52.393826292Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:52400"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:08:52.402592203Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"52160217-34d0-43d5-88a2-10186c6d39be","remote_addr":"[fd00:1122:3344:101::1]:52400","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":8664,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:08:52.40520817Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:08:52.40521879Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:08:54.849050102Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:08:54.849119783Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"aacfdadc-30ed-42b8-a1e1-2d552e57009c","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36196","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":33,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:08:54.982245236Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:08:54.982368047Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:09:04.985116304Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:09:04.985222335Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:09:14.986517076Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:09:14.987037211Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:09:24.882393419Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59982"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:09:24.882464479Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"b5370bb7-f32f-44cc-b33c-ab2217b7bc3c","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59982","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":28,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:09:24.988359372Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:09:24.988476173Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:09:34.991453821Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:09:34.991562282Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:09:44.993251788Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:09:44.993360249Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:09:45.254826558Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:09:45.25502203Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:41054"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:09:45.264671859Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"a4949be6-5ee7-45e9-ab9d-709696159b11","remote_addr":"[fd00:1122:3344:101::1]:41054","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":9570,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:09:45.267559449Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:09:45.267564569Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:09:54.914577564Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56013"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:09:54.914718446Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"04100423-65b4-4029-818a-778b19e13858","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56013","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":42,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:09:54.995997794Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:09:54.996124275Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:10:04.997879872Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:10:04.998071374Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:10:14.999663466Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:10:14.999779237Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:10:24.949316475Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44033"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:10:24.949395316Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"d32f85fb-c39d-4148-9843-1bdaf1cad03d","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44033","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":36,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:10:25.001451843Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:10:25.001574834Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:10:32.007987174Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:10:32.008172546Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:59963"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:10:32.016463491Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"5a8b8db9-9f29-4d14-b4ee-b5fb42ce63a6","remote_addr":"[fd00:1122:3344:101::1]:59963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":8203,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:10:32.01918455Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:10:32.01919421Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:10:35.003796503Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:10:35.003904664Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:10:45.00581819Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:10:45.005941202Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:10:49.670618276Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"142.313519592s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:10:54.981865151Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54246"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:10:54.981936172Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"a2e0b9be-5c4e-4d61-a5aa-2fa093dbd318","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54246","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":30,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:10:55.008182483Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:10:55.008301704Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:11:05.010699106Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:11:05.010800897Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:11:15.012433332Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:11:15.012553673Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:11:25.014583192Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:11:25.014686713Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:11:25.015674163Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56767"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:11:25.015736184Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"b940ddf6-df2b-483d-aa71-8af22afb93cf","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56767","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:867","latency_us":25,"response_code":"200"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:11:33.519882312Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:11:33.520081844Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:65020"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_02996ddd-87ce-45fb-9c94-4c7ed57603e1': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:11:33.529782645Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4020"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-14T20:11:33.529889996Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"e2dbf2a3-3574-49eb-a7bf-4105bf805f02","remote_addr":"[fd00:1122:3344:101::1]:65020","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.15.1/src/server.rs:855","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9730,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-14T20:11:33.532764045Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:11:35.017328817Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:11:35.017447618Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:11:45.019672018Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13594594529322773398"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-14T20:11:45.01978203Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"e20b6ac2-4792-45a8-adc6-0eb48d767a22","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10497610638671551828"}