[ Feb 22 00:54:22 Enabled. ] [ Feb 22 00:54:22 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Feb 22 00:54:22 Method "start" exited with status 0. ] [ Feb 22 00:54:22 Rereading configuration. ] [ Feb 22 00:54:22 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:22.860949783Z","hostname":"buskin","pid":1566,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:22.861116827Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:22.914672827Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:22.914705828Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:23.030779421Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:23.039674454Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:23.04783236Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:23.653141018Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:200::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:23.792943093Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:126"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:23.792966203Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:138"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:23.793056095Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:151"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:23.793130177Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:23.793152358Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:23.793176748Z","hostname":"buskin","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:717"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:23.796112995Z","hostname":"buskin","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:624"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:23.796120265Z","hostname":"buskin","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:766"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:23.796131325Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:187"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:23.796142136Z","hostname":"buskin","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:825"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:23.796168726Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:23.796266708Z","hostname":"buskin","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:23.79631725Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:23.79633809Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:23.79634641Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:23.807960935Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:23.807988906Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"651ad8e9-444b-45dc-9dfc-4c79c25fd16e (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:23.932098653Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 651ad8e9-444b-45dc-9dfc-4c79c25fd16e (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.381440498Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.381469029Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.381475589Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.381486959Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.381494599Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.38150595Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.38150972Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.390470594Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.390487964Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"4a331860-fd72-42e5-801b-f6e26734b6a6 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.497388499Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 4a331860-fd72-42e5-801b-f6e26734b6a6 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943516322Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943560943Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 651ad8e9-444b-45dc-9dfc-4c79c25fd16e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4a331860-fd72-42e5-801b-f6e26734b6a6 (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943601714Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943616134Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943627744Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943643745Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943679025Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943691106Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 651ad8e9-444b-45dc-9dfc-4c79c25fd16e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4a331860-fd72-42e5-801b-f6e26734b6a6 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943707746Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943712536Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943717286Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943727997Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943734927Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943738397Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 651ad8e9-444b-45dc-9dfc-4c79c25fd16e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4a331860-fd72-42e5-801b-f6e26734b6a6 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943754617Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943759047Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943763287Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943768578Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943772688Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943776618Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 651ad8e9-444b-45dc-9dfc-4c79c25fd16e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4a331860-fd72-42e5-801b-f6e26734b6a6 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943792128Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943800688Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943805339Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943809279Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943814519Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943818549Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 651ad8e9-444b-45dc-9dfc-4c79c25fd16e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4a331860-fd72-42e5-801b-f6e26734b6a6 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943839059Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943845809Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943855249Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.94386077Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943905981Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943921271Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943927781Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 651ad8e9-444b-45dc-9dfc-4c79c25fd16e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4a331860-fd72-42e5-801b-f6e26734b6a6 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943951852Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943956072Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943960042Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943963192Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943968042Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943971542Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943976292Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 651ad8e9-444b-45dc-9dfc-4c79c25fd16e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4a331860-fd72-42e5-801b-f6e26734b6a6 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.943997293Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944013503Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944018223Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944043434Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944057224Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944078965Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 651ad8e9-444b-45dc-9dfc-4c79c25fd16e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4a331860-fd72-42e5-801b-f6e26734b6a6 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944113875Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944119866Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944123406Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944126766Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 651ad8e9-444b-45dc-9dfc-4c79c25fd16e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4a331860-fd72-42e5-801b-f6e26734b6a6 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944156467Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944161556Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944164977Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944170527Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944174457Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:97"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944183137Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:99"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944186557Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 651ad8e9-444b-45dc-9dfc-4c79c25fd16e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4a331860-fd72-42e5-801b-f6e26734b6a6 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944212098Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944216678Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944219788Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944222798Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:211"} {"msg":"No ledger in /pool/int/651ad8e9-444b-45dc-9dfc-4c79c25fd16e/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944225838Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/4a331860-fd72-42e5-801b-f6e26734b6a6/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944232818Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944240178Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/651ad8e9-444b-45dc-9dfc-4c79c25fd16e/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944253939Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/4a331860-fd72-42e5-801b-f6e26734b6a6/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944257599Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944260769Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.944284049Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:216"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.94428795Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:231"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.954936402Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:793"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.954961722Z","hostname":"buskin","pid":1566,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999947839s","next_cleanup":"Instant { tv_sec: 1163, tv_nsec: 815841239 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.954990673Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4018"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.955034544Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/651ad8e9-444b-45dc-9dfc-4c79c25fd16e/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.957601033Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/4a331860-fd72-42e5-801b-f6e26734b6a6/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.957647544Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.957980011Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:328"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.959620039Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:51","local_addr":"[fdb0:200::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: Some(\n None,\n ),\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n tx_eq: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.960086279Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1190"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.960094259Z","hostname":"buskin","pid":1566,"component":"DnsResolver","component":"RSS","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.96010073Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.960173401Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/651ad8e9-444b-45dc-9dfc-4c79c25fd16e/config/rss-started.marker","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.960201872Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/4a331860-fd72-42e5-801b-f6e26734b6a6/config/rss-started.marker","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.960212742Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/651ad8e9-444b-45dc-9dfc-4c79c25fd16e/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.960225713Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/4a331860-fd72-42e5-801b-f6e26734b6a6/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.960234173Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS not previously run. Creating plans.","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.960245913Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1239"} {"msg":"Writing ledger to /pool/int/651ad8e9-444b-45dc-9dfc-4c79c25fd16e/config/.rss-started.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.960260533Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/4a331860-fd72-42e5-801b-f6e26734b6a6/config/.rss-started.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.960465858Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.960621462Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1271"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.960626142Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:66"} {"msg":"Creating plan for the sled at fdb0:200::1","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.960629622Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:39"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.960687063Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1302"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.960762835Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/651ad8e9-444b-45dc-9dfc-4c79c25fd16e/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.960771815Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/4a331860-fd72-42e5-801b-f6e26734b6a6/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.960940369Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.961077992Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.961188664Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:200::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 46437054-bf3d-4063-a6ee-f11c748dcba3 (sled), rack_id: c563b7c0-6a47-4234-b338-2f9bf972dbe4, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.962003423Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4315"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.962063364Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4721"} {"msg":"Loaded keys and certs","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.966317261Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:200::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-882d17aeeb0cb343/cc13773/tls/src/client.rs:50"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.972009771Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:200::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-882d17aeeb0cb343/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.972170864Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:200::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-882d17aeeb0cb343/cc13773/tls/src/keys.rs:376"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.976874961Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-882d17aeeb0cb343/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.976939343Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-882d17aeeb0cb343/cc13773/tls/src/keys.rs:376"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.977035595Z","hostname":"buskin","pid":1566,"remote_addr":"[fdb0:200::1]:33339","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:74"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 46437054-bf3d-4063-a6ee-f11c748dcba3 (sled), rack_id: c563b7c0-6a47-4234-b338-2f9bf972dbe4, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.977181688Z","hostname":"buskin","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:362"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.977195689Z","hostname":"buskin","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:377"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.97723304Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/server.rs:44"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.97723928Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.97724381Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:813"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.97725249Z","hostname":"buskin","pid":1566,"component":"DnsResolver","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"No ledger in /pool/int/651ad8e9-444b-45dc-9dfc-4c79c25fd16e/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.977274281Z","hostname":"buskin","pid":1566,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.977279041Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"No ledger in /pool/int/4a331860-fd72-42e5-801b-f6e26734b6a6/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.977293761Z","hostname":"buskin","pid":1566,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.977300301Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:802"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.977308631Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:830"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.977313871Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:596"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.977383113Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.977389603Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:701","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:24.99437838Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.015281926Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.028580539Z","hostname":"buskin","pid":1566,"sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:419"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.02862205Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.02863042Z","hostname":"buskin","pid":1566,"sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:431"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.028647931Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 112.206196ms)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.031533357Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7817e7a2-3149-44f7-a38d-5ddf7581af73\", \"content-length\": \"124\", \"date\": \"Sat, 22 Feb 2025 00:54:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7817e7a2-3149-44f7-a38d-5ddf7581af73\" }"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.032636832Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"attempting to create encrypted zvol: name=\"oxi_651ad8e9-444b-45dc-9dfc-4c79c25fd16e/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.041538194Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_651ad8e9-444b-45dc-9dfc-4c79c25fd16e/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.072211923Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_651ad8e9-444b-45dc-9dfc-4c79c25fd16e/swap\"","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.072253904Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.077104045Z","hostname":"buskin","pid":1566,"sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:446"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.077113185Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:129"} {"msg":"Ensuring dataset oxi_651ad8e9-444b-45dc-9dfc-4c79c25fd16e/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.077118035Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:139"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.145589165Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 130.739996ms)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.146491045Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b81066ef-d07f-4f3d-b370-187fedccd833\", \"content-length\": \"124\", \"date\": \"Sat, 22 Feb 2025 00:54:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b81066ef-d07f-4f3d-b370-187fedccd833\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.152694877Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:174"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.278903501Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 188.689722ms)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.279612198Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c1d80a1b-36d3-47bd-8c61-2be5f2429d52\", \"content-length\": \"124\", \"date\": \"Sat, 22 Feb 2025 00:54:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c1d80a1b-36d3-47bd-8c61-2be5f2429d52\" }"} {"msg":"Mounting oxi_651ad8e9-444b-45dc-9dfc-4c79c25fd16e/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.371974872Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:182"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.385409327Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:190"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.385537461Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:190"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.385573491Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:190"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.385607072Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:196"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.469224467Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 123.278572ms)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.469934743Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"accb403e-64f4-4160-bf37-89bd384290ad\", \"content-length\": \"124\", \"date\": \"Sat, 22 Feb 2025 00:54:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"accb403e-64f4-4160-bf37-89bd384290ad\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.595038603Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 268.558926ms)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.595719338Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ee9f9061-6a65-4d82-9bb1-34a7925dd76c\", \"content-length\": \"124\", \"date\": \"Sat, 22 Feb 2025 00:54:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ee9f9061-6a65-4d82-9bb1-34a7925dd76c\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.865814721Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 194.971986ms)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:25.86666238Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"53d93e59-3542-4bb7-82ff-ff15442cfff4\", \"content-length\": \"124\", \"date\": \"Sat, 22 Feb 2025 00:54:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"53d93e59-3542-4bb7-82ff-ff15442cfff4\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:26.062732876Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 221.518444ms)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:26.063370561Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"93db81cb-b8e2-4c80-8120-1f55d0739094\", \"content-length\": \"124\", \"date\": \"Sat, 22 Feb 2025 00:54:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"93db81cb-b8e2-4c80-8120-1f55d0739094\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:26.136016966Z","hostname":"buskin","pid":1566,"sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:94"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:26.141703345Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:58525","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:328","component":"producer-server","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:328"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:26.141746416Z","hostname":"buskin","pid":1566,"component":"internal-dns-resolver","file":"internal-dns/resolver/src/resolver.rs:111","component":"producer-server","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:26.141800467Z","hostname":"buskin","pid":1566,"file":"oximeter/producer/src/lib.rs:279","component":"producer-server","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:279","interval":"30s","address":"[fd00:1122:3344:101::1]:58525","producer_id":"46437054-bf3d-4063-a6ee-f11c748dcba3"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:26.142910243Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"282.098441ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:26.143987647Z","hostname":"buskin","pid":1566,"component":"VmmReservoirManager","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:26.143997737Z","hostname":"buskin","pid":1566,"component":"VmmReservoirManager","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:26.144056989Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:977","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:26.144063449Z","hostname":"buskin","pid":1566,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:26.285861179Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:26.42636745Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"323.759318ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:26.752156841Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.415028904s"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:28.169109537Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.551223824s"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:29.221575792Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2749"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:29.221604092Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2977"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:29.221609103Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2619"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:29.221622813Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2666"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:29.221666414Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:29.221671384Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3103"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:29.223582528Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2885"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:29.224934618Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2660"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:29.225050041Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:29.225239735Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:489"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_switch\") failed: Property not found. retry in 32.637365ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:29.620604071Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_switch\") failed: Property not found. retry in 137.381474ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:29.825828186Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_switch\") failed: Property not found. retry in 255.07457ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:30.025517075Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_switch\") failed: Property not found. retry in 268.062914ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:30.422427626Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_switch\") failed: Property not found. retry in 709.065634ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:30.878302481Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:30.879132689Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"5.668483998s"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_switch\") failed: Property not found. retry in 1.343585907s","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:31.732513299Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_switch\") failed: Property not found. retry in 891.93516ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:33.148614905Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_switch\") failed: Property not found. retry in 857.444397ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:34.242858811Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.47856281Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.479100552Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12348","component":"dropshot (Repo Depot)","component":"ArtifactStore","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:328"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.480461973Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:328"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.480495543Z","hostname":"buskin","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:432"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.480508244Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Writing ledger to /pool/int/651ad8e9-444b-45dc-9dfc-4c79c25fd16e/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.480521614Z","hostname":"buskin","pid":1566,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/4a331860-fd72-42e5-801b-f6e26734b6a6/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.480702508Z","hostname":"buskin","pid":1566,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.480874992Z","hostname":"buskin","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.480929113Z","hostname":"buskin","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.1.100, vlan_id: None, rib_priority: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: Some(None), bgp_peers: [], autoneg: false, lldp: None, tx_eq: None }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.480937914Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4018"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.480974885Z","hostname":"buskin","pid":1566,"remote_addr":"[fdb0:200::1]:33339","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:81"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.672291632Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:117","peer":"[fdb0:200::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.672360164Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:252","target_sled":"[fdb0:200::1]:12346"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.677620454Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4339","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.677642084Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4355"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.677646725Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:535"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.916293721Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:63070"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.916550436Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.916557747Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/651ad8e9-444b-45dc-9dfc-4c79c25fd16e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.916638238Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/4a331860-fd72-42e5-801b-f6e26734b6a6/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.916651019Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.916776772Z","hostname":"buskin","pid":1566,"uri":"/inventory","method":"GET","req_id":"8a409dc6-92ac-45ff-a4c7-46d4435aa977","remote_addr":"[fd00:1122:3344:101::1]:63070","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":347,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.946839057Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:57813"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.947166324Z","hostname":"buskin","pid":1566,"uri":"/sled-role","method":"GET","req_id":"afc69833-c31d-4190-8efa-efc306a2846c","remote_addr":"[fd00:1122:3344:101::1]:57813","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":79,"response_code":"200"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(2), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 0007e466-1291-455e-826e-5f9e2f7afb0b (physical_disk), pool_id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 1ac047db-3455-453f-b4b5-e7bbf61d5d8a (physical_disk), pool_id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 53df1f61-f504-4ae9-a220-a3e5cc074986 (physical_disk), pool_id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 5ff25c66-4cf6-4cf4-aed3-606cffb821d7 (physical_disk), pool_id: bc2d7f49-c833-4446-a01d-161c755b095b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 6480b7f0-d228-4048-b716-ab4bfe7895dd (physical_disk), pool_id: 3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 75cc4a81-3fe2-47ea-941e-5710abb25666 (physical_disk), pool_id: 6a73c9fc-2f58-450b-97e5-4ef356bb98fb (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: a9419049-9194-480a-904c-ccb1e4e09d48 (physical_disk), pool_id: b478d3fc-de0b-4188-973f-7d9bae6f3554 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: b39ccd93-ec57-45cd-aa73-97f89c03fd71 (physical_disk), pool_id: 7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: ce660079-5780-4e5f-ace1-f98bee145475 (physical_disk), pool_id: 7dd4c27f-a9f0-4c2e-a0e0-1ad507079428 (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.97817456Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:385"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.978279373Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:48101"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.978508218Z","hostname":"buskin","pid":1566,"sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:895"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(2), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 0007e466-1291-455e-826e-5f9e2f7afb0b (physical_disk), pool_id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 1ac047db-3455-453f-b4b5-e7bbf61d5d8a (physical_disk), pool_id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 53df1f61-f504-4ae9-a220-a3e5cc074986 (physical_disk), pool_id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 5ff25c66-4cf6-4cf4-aed3-606cffb821d7 (physical_disk), pool_id: bc2d7f49-c833-4446-a01d-161c755b095b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 6480b7f0-d228-4048-b716-ab4bfe7895dd (physical_disk), pool_id: 3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 75cc4a81-3fe2-47ea-941e-5710abb25666 (physical_disk), pool_id: 6a73c9fc-2f58-450b-97e5-4ef356bb98fb (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: a9419049-9194-480a-904c-ccb1e4e09d48 (physical_disk), pool_id: b478d3fc-de0b-4188-973f-7d9bae6f3554 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: b39ccd93-ec57-45cd-aa73-97f89c03fd71 (physical_disk), pool_id: 7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: ce660079-5780-4e5f-ace1-f98bee145475 (physical_disk), pool_id: 7dd4c27f-a9f0-4c2e-a0e0-1ad507079428 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.978540919Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/651ad8e9-444b-45dc-9dfc-4c79c25fd16e/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.978549149Z","hostname":"buskin","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/4a331860-fd72-42e5-801b-f6e26734b6a6/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.978554099Z","hostname":"buskin","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.978561379Z","hostname":"buskin","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1298"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.978593Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.97859815Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.97860284Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:35.97860652Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.118914436Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.118944177Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool)"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:54:36.322978495Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:54:36.323106048Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.524875564Z","hostname":"buskin","pid":1566,"component":"VmmReservoirManager","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/46437054-bf3d-4063-a6ee-f11c748dcba3)","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:54:36.526703115Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:238"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:36.549386192Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"7.043123882s"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.593240351Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.601219733Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.601274764Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.601282894Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.601287544Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.601293114Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.601404527Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.601409837Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.643639799Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.65114121Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.823875465Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.831759234Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.831789795Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.831811785Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.831834886Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.831839556Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.841882805Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.841911866Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.951347979Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.958914741Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.958977322Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.958985262Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.959038624Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.959049894Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.959217708Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:36.959224208Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.003716631Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.011639891Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.182449402Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.190530436Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.190560827Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.190566747Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.190573767Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.190578697Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.200183976Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.200205297Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"f628b386-82f1-43d6-bae7-1b54d721271c (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.31227545Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.320272772Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.320311333Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.320323323Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.320353254Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.320360934Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.320486617Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.320492557Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.361372578Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.369285678Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/46437054-bf3d-4063-a6ee-f11c748dcba3)","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:54:37.529015707Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:238"} {"msg":"Looking for unencrypted datasets in oxp_f628b386-82f1-43d6-bae7-1b54d721271c","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.537139802Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.545077683Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.545102923Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.545108633Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.545114853Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.545119714Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.554336664Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.554361014Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"b478d3fc-de0b-4188-973f-7d9bae6f3554 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.667094912Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: b478d3fc-de0b-4188-973f-7d9bae6f3554 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.674992702Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.675034333Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.675044703Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.675099985Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.675121405Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.675303619Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.67531541Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.717616563Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_b478d3fc-de0b-4188-973f-7d9bae6f3554/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.725281758Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_b478d3fc-de0b-4188-973f-7d9bae6f3554","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.896260722Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.904271435Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.904306676Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.904312646Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.904317926Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.904322766Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.913924605Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:37.913945765Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"bc2d7f49-c833-4446-a01d-161c755b095b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.024214577Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: bc2d7f49-c833-4446-a01d-161c755b095b (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.032071456Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.032109497Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.032116417Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.032155048Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.032166388Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.032317482Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.032328382Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.07394699Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_bc2d7f49-c833-4446-a01d-161c755b095b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.081678906Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_bc2d7f49-c833-4446-a01d-161c755b095b","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.248377683Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.256217692Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.256268913Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.256277953Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.256286003Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.256294013Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.265890522Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.265915873Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"7dd4c27f-a9f0-4c2e-a0e0-1ad507079428 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.378699132Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: 7dd4c27f-a9f0-4c2e-a0e0-1ad507079428 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.386621952Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.386650753Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.386685244Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.386730915Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.386744085Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.386881608Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.386890608Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.427261578Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_7dd4c27f-a9f0-4c2e-a0e0-1ad507079428/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.435170908Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/46437054-bf3d-4063-a6ee-f11c748dcba3)","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:54:38.530836257Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:238"} {"msg":"Looking for unencrypted datasets in oxp_7dd4c27f-a9f0-4c2e-a0e0-1ad507079428","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.608296802Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.616397356Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.616442897Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.616467518Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.616476958Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.616483298Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.6262267Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.626260121Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"6a73c9fc-2f58-450b-97e5-4ef356bb98fb (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.742659083Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 6a73c9fc-2f58-450b-97e5-4ef356bb98fb (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.750064901Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.750100642Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.750112212Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.750164823Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.750179094Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.750298977Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.750305887Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.795177219Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_6a73c9fc-2f58-450b-97e5-4ef356bb98fb/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:38.803004927Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_6a73c9fc-2f58-450b-97e5-4ef356bb98fb","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.001350705Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.009433699Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.00946974Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.00947719Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.00948574Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.00949157Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.019486128Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.019513449Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.139795699Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.148036886Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.148079597Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.148098318Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.148119028Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.148136949Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.148286952Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.148293342Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.189990142Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.197771139Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.368402276Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.376265475Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.376288236Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.376293946Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.376298136Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.376301776Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.386297024Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.386319284Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.501971409Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.509850018Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.509873069Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.509878009Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.509947101Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.509953721Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.510095204Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.510100934Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/46437054-bf3d-4063-a6ee-f11c748dcba3)","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:54:39.533322613Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:238"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.558444425Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.5660928Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.763910146Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.771851416Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.771897468Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/651ad8e9-444b-45dc-9dfc-4c79c25fd16e/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.77201087Z","hostname":"buskin","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.772206324Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(2), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 651ad8e9-444b-45dc-9dfc-4c79c25fd16e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4a331860-fd72-42e5-801b-f6e26734b6a6 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: b478d3fc-de0b-4188-973f-7d9bae6f3554 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: bc2d7f49-c833-4446-a01d-161c755b095b (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 7dd4c27f-a9f0-4c2e-a0e0-1ad507079428 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 6a73c9fc-2f58-450b-97e5-4ef356bb98fb (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6 (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.772291886Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.772302927Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/4a331860-fd72-42e5-801b-f6e26734b6a6/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.772361158Z","hostname":"buskin","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.772552403Z","hostname":"buskin","pid":1566,"sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:899"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.772559843Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.772566263Z","hostname":"buskin","pid":1566,"sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:925","generation":"Generation(2)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.772570783Z","hostname":"buskin","pid":1566,"sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:930"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.772575873Z","hostname":"buskin","pid":1566,"sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:935"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.772608784Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:123","gen":"Generation(2)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.772617544Z","hostname":"buskin","pid":1566,"sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:941"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.772643494Z","hostname":"buskin","pid":1566,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:799","gen":"Generation(2)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.772650085Z","hostname":"buskin","pid":1566,"sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:946"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.772717696Z","hostname":"buskin","pid":1566,"uri":"/omicron-physical-disks","method":"PUT","req_id":"7c6de85e-a75d-43f2-98e3-9a4eef8648c0","remote_addr":"[fd00:1122:3344:101::1]:48101","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":3794352,"response_code":"200"} {"msg":"attempting to set up sled's datasets: DatasetsConfig { generation: Generation(1), datasets: {02e6e221-7e0d-49b2-a577-4b416e81ba1c (dataset): DatasetConfig { id: 02e6e221-7e0d-49b2-a577-4b416e81ba1c (dataset), name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: TransientZone { name: \"oxz_clickhouse_4d4b2424-58da-49f4-8f26-04c0d70d01b3\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 061be0ca-5cd6-4179-98df-a39b9a4ff22d (dataset): DatasetConfig { id: 061be0ca-5cd6-4179-98df-a39b9a4ff22d (dataset), name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_1c81cbcf-1fc4-4d48-a274-adee5ad7eaa2\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0abce1a4-a093-460b-ba02-5371afebcf04 (dataset): DatasetConfig { id: 0abce1a4-a093-460b-ba02-5371afebcf04 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0f4be58f-7d14-4e9e-ac31-91fa9ffe782a (dataset): DatasetConfig { id: 0f4be58f-7d14-4e9e-ac31-91fa9ffe782a (dataset), name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 15bd5b79-be2e-4233-94de-51f353197dbe (dataset): DatasetConfig { id: 15bd5b79-be2e-4233-94de-51f353197dbe (dataset), name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_754b7158-e16d-4a6a-a0bc-b68f75a4910b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 161e12fc-f92d-4c75-9df8-d97eab609b47 (dataset): DatasetConfig { id: 161e12fc-f92d-4c75-9df8-d97eab609b47 (dataset), name: DatasetName { pool_name: ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 16217110-555d-4be4-b912-42acac7da84e (dataset): DatasetConfig { id: 16217110-555d-4be4-b912-42acac7da84e (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_b87d7da8-d424-414d-8ec7-c372ad128c96\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 173c1050-706f-4284-b7e6-7bb7ebe61868 (dataset): DatasetConfig { id: 173c1050-706f-4284-b7e6-7bb7ebe61868 (dataset), name: DatasetName { pool_name: ZpoolName { id: b478d3fc-de0b-4188-973f-7d9bae6f3554 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_f583c1f5-9142-4c11-9241-e867fa84cb28\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 25ae3bcb-1fe7-4410-a154-49e966f1dff6 (dataset): DatasetConfig { id: 25ae3bcb-1fe7-4410-a154-49e966f1dff6 (dataset), name: DatasetName { pool_name: ZpoolName { id: b478d3fc-de0b-4188-973f-7d9bae6f3554 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 28d3901e-7c34-4f25-9e46-af8d0ab8f21e (dataset): DatasetConfig { id: 28d3901e-7c34-4f25-9e46-af8d0ab8f21e (dataset), name: DatasetName { pool_name: ZpoolName { id: 7dd4c27f-a9f0-4c2e-a0e0-1ad507079428 (zpool), kind: External }, kind: TransientZone { name: \"oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 28f8668a-801d-4618-86a2-1c52105ab565 (dataset): DatasetConfig { id: 28f8668a-801d-4618-86a2-1c52105ab565 (dataset), name: DatasetName { pool_name: ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2c427402-a4a6-485a-b0e8-ad487b2fcb34 (dataset): DatasetConfig { id: 2c427402-a4a6-485a-b0e8-ad487b2fcb34 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_85387c78-aa29-49a7-9f9e-3490ed51f967\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2e9bb792-6540-4648-acb7-808bc465d89c (dataset): DatasetConfig { id: 2e9bb792-6540-4648-acb7-808bc465d89c (dataset), name: DatasetName { pool_name: ZpoolName { id: 7dd4c27f-a9f0-4c2e-a0e0-1ad507079428 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2f7e5109-2165-4ab7-8f7c-b02d367f8705 (dataset): DatasetConfig { id: 2f7e5109-2165-4ab7-8f7c-b02d367f8705 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6 (zpool), kind: External }, kind: TransientZone { name: \"oxz_oximeter_7b35ff06-c97e-4b81-8f7a-0e6c7999e8ed\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3131cb23-868d-429a-883a-1d920e1232a4 (dataset): DatasetConfig { id: 3131cb23-868d-429a-883a-1d920e1232a4 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_e90d0163-99b4-4559-bdb8-a40ce4a2ffa9\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 340a6ad3-147f-4a58-8bad-a6b4dd952360 (dataset): DatasetConfig { id: 340a6ad3-147f-4a58-8bad-a6b4dd952360 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 34fbe576-016a-4fa2-95cf-a5f1d913ca6c (dataset): DatasetConfig { id: 34fbe576-016a-4fa2-95cf-a5f1d913ca6c (dataset), name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3a8f21f5-ef87-43ed-b512-21015ac4a8cf (dataset): DatasetConfig { id: 3a8f21f5-ef87-43ed-b512-21015ac4a8cf (dataset), name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3d7f7b91-431a-4af5-a3f3-196a783c869e (dataset): DatasetConfig { id: 3d7f7b91-431a-4af5-a3f3-196a783c869e (dataset), name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_016d571e-a5ea-4351-b665-3e607b682e72\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 401ff6a9-f4d1-4dca-ab67-80e72f8b4577 (dataset): DatasetConfig { id: 401ff6a9-f4d1-4dca-ab67-80e72f8b4577 (dataset), name: DatasetName { pool_name: ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_780e66c3-354e-48ea-b337-3b7eceac6dc0\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4c720ccb-ae25-4382-a0eb-b2cbe382be06 (dataset): DatasetConfig { id: 4c720ccb-ae25-4382-a0eb-b2cbe382be06 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc2d7f49-c833-4446-a01d-161c755b095b (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4c7f7eca-443b-4990-9ef0-34c08e72b537 (dataset): DatasetConfig { id: 4c7f7eca-443b-4990-9ef0-34c08e72b537 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 530cdfe3-7cd6-4bbb-9365-307690fc6df9 (dataset): DatasetConfig { id: 530cdfe3-7cd6-4bbb-9365-307690fc6df9 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_6b344173-8cba-4e37-b5e5-362444c7655b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 577802f9-8f3a-4f07-8df3-136baab358f5 (dataset): DatasetConfig { id: 577802f9-8f3a-4f07-8df3-136baab358f5 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7dd4c27f-a9f0-4c2e-a0e0-1ad507079428 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 5cbc858d-2d4d-4fb5-a949-9c3c0bbf29df (dataset): DatasetConfig { id: 5cbc858d-2d4d-4fb5-a949-9c3c0bbf29df (dataset), name: DatasetName { pool_name: ZpoolName { id: 6a73c9fc-2f58-450b-97e5-4ef356bb98fb (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 5f29730a-e1bd-4849-b3e9-7f3f6e7b62a1 (dataset): DatasetConfig { id: 5f29730a-e1bd-4849-b3e9-7f3f6e7b62a1 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc2d7f49-c833-4446-a01d-161c755b095b (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5f80f3b8-3a5c-4e2f-bac2-5131cc6d1961 (dataset): DatasetConfig { id: 5f80f3b8-3a5c-4e2f-bac2-5131cc6d1961 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc2d7f49-c833-4446-a01d-161c755b095b (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_f7f17c9a-0027-41cc-b185-223fb703772f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 692ca293-5453-4726-97c9-ab0949f77fad (dataset): DatasetConfig { id: 692ca293-5453-4726-97c9-ab0949f77fad (dataset), name: DatasetName { pool_name: ZpoolName { id: 6a73c9fc-2f58-450b-97e5-4ef356bb98fb (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6b9f94dc-3eb0-4d70-beb2-b69c9252013b (dataset): DatasetConfig { id: 6b9f94dc-3eb0-4d70-beb2-b69c9252013b (dataset), name: DatasetName { pool_name: ZpoolName { id: 7dd4c27f-a9f0-4c2e-a0e0-1ad507079428 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 72ee2981-8311-437c-9462-84cd0aa7894b (dataset): DatasetConfig { id: 72ee2981-8311-437c-9462-84cd0aa7894b (dataset), name: DatasetName { pool_name: ZpoolName { id: 6a73c9fc-2f58-450b-97e5-4ef356bb98fb (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_71d88fcd-cb6a-4be5-9a21-e924eee85be6\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 74456b27-84f1-4405-8925-03d0c3daaf91 (dataset): DatasetConfig { id: 74456b27-84f1-4405-8925-03d0c3daaf91 (dataset), name: DatasetName { pool_name: ZpoolName { id: 6a73c9fc-2f58-450b-97e5-4ef356bb98fb (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 80b0bb64-5003-4140-a3ae-e728de4e6520 (dataset): DatasetConfig { id: 80b0bb64-5003-4140-a3ae-e728de4e6520 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_f1598ca1-8c9f-4830-9931-a5c2d7746097\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8179d1df-ac1b-4591-819d-b435b32912ff (dataset): DatasetConfig { id: 8179d1df-ac1b-4591-819d-b435b32912ff (dataset), name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 8a665284-3bfc-4da5-93a5-5185366ab28f (dataset): DatasetConfig { id: 8a665284-3bfc-4da5-93a5-5185366ab28f (dataset), name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 92000373-7eed-4d74-a43a-17556b1c575a (dataset): DatasetConfig { id: 92000373-7eed-4d74-a43a-17556b1c575a (dataset), name: DatasetName { pool_name: ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9cc0b486-5e37-481a-9056-b816922e4f6b (dataset): DatasetConfig { id: 9cc0b486-5e37-481a-9056-b816922e4f6b (dataset), name: DatasetName { pool_name: ZpoolName { id: bc2d7f49-c833-4446-a01d-161c755b095b (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_d4420bcb-68b6-430d-b904-17ad9aef41ba\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9d06b2cc-fe6c-46b1-8212-855979a47b0d (dataset): DatasetConfig { id: 9d06b2cc-fe6c-46b1-8212-855979a47b0d (dataset), name: DatasetName { pool_name: ZpoolName { id: bc2d7f49-c833-4446-a01d-161c755b095b (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9ee73f03-a978-42f0-b407-b38cb4b8a8ff (dataset): DatasetConfig { id: 9ee73f03-a978-42f0-b407-b38cb4b8a8ff (dataset), name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, a098cdbb-70da-4d36-be68-aa314b750bd0 (dataset): DatasetConfig { id: a098cdbb-70da-4d36-be68-aa314b750bd0 (dataset), name: DatasetName { pool_name: ZpoolName { id: b478d3fc-de0b-4188-973f-7d9bae6f3554 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a180f7ba-3a40-4377-91b8-9a8c85f14c1e (dataset): DatasetConfig { id: a180f7ba-3a40-4377-91b8-9a8c85f14c1e (dataset), name: DatasetName { pool_name: ZpoolName { id: bc2d7f49-c833-4446-a01d-161c755b095b (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, a1ca7c88-ad96-4060-b48b-6a7c079b3bd4 (dataset): DatasetConfig { id: a1ca7c88-ad96-4060-b48b-6a7c079b3bd4 (dataset), name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_93524fa4-4882-458b-a91f-54ec48cb5768\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a253db52-2f3b-4246-bd95-d4d684185c22 (dataset): DatasetConfig { id: a253db52-2f3b-4246-bd95-d4d684185c22 (dataset), name: DatasetName { pool_name: ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a3fd4fdf-f354-4385-910c-df2740ca61f6 (dataset): DatasetConfig { id: a3fd4fdf-f354-4385-910c-df2740ca61f6 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, ac8df8ce-67ef-41a1-b5f6-3fdbcc21e575 (dataset): DatasetConfig { id: ac8df8ce-67ef-41a1-b5f6-3fdbcc21e575 (dataset), name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ae82aeb1-d80c-4b4a-a588-6dbf6b264bd2 (dataset): DatasetConfig { id: ae82aeb1-d80c-4b4a-a588-6dbf6b264bd2 (dataset), name: DatasetName { pool_name: ZpoolName { id: b478d3fc-de0b-4188-973f-7d9bae6f3554 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_e506936b-b474-4691-885c-9f5f291e9b52\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b26b5777-dfb9-4ca3-a0cd-d6629181b00b (dataset): DatasetConfig { id: b26b5777-dfb9-4ca3-a0cd-d6629181b00b (dataset), name: DatasetName { pool_name: ZpoolName { id: 7dd4c27f-a9f0-4c2e-a0e0-1ad507079428 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_fae1a638-2f10-4b23-9417-1d370e042bb5\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b2e23b21-2efb-42cf-b657-13d8a8f6d10e (dataset): DatasetConfig { id: b2e23b21-2efb-42cf-b657-13d8a8f6d10e (dataset), name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, bc8cd983-8247-429a-b32b-9955a13910cd (dataset): DatasetConfig { id: bc8cd983-8247-429a-b32b-9955a13910cd (dataset), name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_e43e6ae4-7893-4170-ad06-c06d9fd9de91\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, be6889a8-2277-426f-b49f-95b22c72944d (dataset): DatasetConfig { id: be6889a8-2277-426f-b49f-95b22c72944d (dataset), name: DatasetName { pool_name: ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, c4b9c223-e09f-4fe1-86ab-44491d862baf (dataset): DatasetConfig { id: c4b9c223-e09f-4fe1-86ab-44491d862baf (dataset), name: DatasetName { pool_name: ZpoolName { id: 7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_c3f170b5-2656-46a9-9552-0acd23b28479\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c5e2b4e8-3414-4550-b8a3-93b1fdc4c76d (dataset): DatasetConfig { id: c5e2b4e8-3414-4550-b8a3-93b1fdc4c76d (dataset), name: DatasetName { pool_name: ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_d993ecda-7c61-4422-b656-98f08a42bf37\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c8669db1-e304-4bd9-8bdb-2fa3899eff78 (dataset): DatasetConfig { id: c8669db1-e304-4bd9-8bdb-2fa3899eff78 (dataset), name: DatasetName { pool_name: ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ce0571b9-b5fb-4253-88ce-f65b4d5f15ea (dataset): DatasetConfig { id: ce0571b9-b5fb-4253-88ce-f65b4d5f15ea (dataset), name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d807188a-ad42-42e8-9d0d-ca352e77599b (dataset): DatasetConfig { id: d807188a-ad42-42e8-9d0d-ca352e77599b (dataset), name: DatasetName { pool_name: ZpoolName { id: 7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d88807bb-10cb-4670-af05-0479a7fb1fbd (dataset): DatasetConfig { id: d88807bb-10cb-4670-af05-0479a7fb1fbd (dataset), name: DatasetName { pool_name: ZpoolName { id: b478d3fc-de0b-4188-973f-7d9bae6f3554 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, db3c74ba-01e4-42e1-a228-e56424323a3a (dataset): DatasetConfig { id: db3c74ba-01e4-42e1-a228-e56424323a3a (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, decdff62-64c0-48d0-97bd-4fad4c5a654b (dataset): DatasetConfig { id: decdff62-64c0-48d0-97bd-4fad4c5a654b (dataset), name: DatasetName { pool_name: ZpoolName { id: 7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_4f2d90dc-0f1a-4fef-a6eb-7e9430c6f5cb\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e70ebeda-90a5-47ad-9548-1ec6610a5901 (dataset): DatasetConfig { id: e70ebeda-90a5-47ad-9548-1ec6610a5901 (dataset), name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e723210d-fa11-4093-b06d-08bb368024fd (dataset): DatasetConfig { id: e723210d-fa11-4093-b06d-08bb368024fd (dataset), name: DatasetName { pool_name: ZpoolName { id: b478d3fc-de0b-4188-973f-7d9bae6f3554 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_407c6397-e5e8-474a-82a5-2ddcd6a5fbb1\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e7e00d8c-a667-443a-a692-9215971872cb (dataset): DatasetConfig { id: e7e00d8c-a667-443a-a692-9215971872cb (dataset), name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, eca1f5d3-ab9d-4603-9252-d98bd759fc44 (dataset): DatasetConfig { id: eca1f5d3-ab9d-4603-9252-d98bd759fc44 (dataset), name: DatasetName { pool_name: ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_80a37bc6-efc5-46e2-8f04-a801f379f3c2\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ed7184cd-4031-4833-b0b1-927a52e3f86b (dataset): DatasetConfig { id: ed7184cd-4031-4833-b0b1-927a52e3f86b (dataset), name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f7d057c6-023b-4c28-a9fa-b41942b56c16 (dataset): DatasetConfig { id: f7d057c6-023b-4c28-a9fa-b41942b56c16 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f955c46d-6b26-42f6-bcb5-4d98ba67fe79 (dataset): DatasetConfig { id: f955c46d-6b26-42f6-bcb5-4d98ba67fe79 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fb67d044-b12d-4703-99d5-4e2b2ebfb5f8 (dataset): DatasetConfig { id: fb67d044-b12d-4703-99d5-4e2b2ebfb5f8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fee779be-7f25-4ef4-8636-422becca3d95 (dataset): DatasetConfig { id: fee779be-7f25-4ef4-8636-422becca3d95 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }} }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.80451067Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:460"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.804614303Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:36955"} {"msg":"datasets ensure","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.805174126Z","hostname":"buskin","pid":1566,"sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:861"} {"msg":"Received DatasetsEnsure { config: DatasetsConfig { generation: Generation(1), datasets: {02e6e221-7e0d-49b2-a577-4b416e81ba1c (dataset): DatasetConfig { id: 02e6e221-7e0d-49b2-a577-4b416e81ba1c (dataset), name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: TransientZone { name: \"oxz_clickhouse_4d4b2424-58da-49f4-8f26-04c0d70d01b3\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 061be0ca-5cd6-4179-98df-a39b9a4ff22d (dataset): DatasetConfig { id: 061be0ca-5cd6-4179-98df-a39b9a4ff22d (dataset), name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_1c81cbcf-1fc4-4d48-a274-adee5ad7eaa2\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0abce1a4-a093-460b-ba02-5371afebcf04 (dataset): DatasetConfig { id: 0abce1a4-a093-460b-ba02-5371afebcf04 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0f4be58f-7d14-4e9e-ac31-91fa9ffe782a (dataset): DatasetConfig { id: 0f4be58f-7d14-4e9e-ac31-91fa9ffe782a (dataset), name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 15bd5b79-be2e-4233-94de-51f353197dbe (dataset): DatasetConfig { id: 15bd5b79-be2e-4233-94de-51f353197dbe (dataset), name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_754b7158-e16d-4a6a-a0bc-b68f75a4910b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 161e12fc-f92d-4c75-9df8-d97eab609b47 (dataset): DatasetConfig { id: 161e12fc-f92d-4c75-9df8-d97eab609b47 (dataset), name: DatasetName { pool_name: ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 16217110-555d-4be4-b912-42acac7da84e (dataset): DatasetConfig { id: 16217110-555d-4be4-b912-42acac7da84e (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_b87d7da8-d424-414d-8ec7-c372ad128c96\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 173c1050-706f-4284-b7e6-7bb7ebe61868 (dataset): DatasetConfig { id: 173c1050-706f-4284-b7e6-7bb7ebe61868 (dataset), name: DatasetName { pool_name: ZpoolName { id: b478d3fc-de0b-4188-973f-7d9bae6f3554 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_f583c1f5-9142-4c11-9241-e867fa84cb28\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 25ae3bcb-1fe7-4410-a154-49e966f1dff6 (dataset): DatasetConfig { id: 25ae3bcb-1fe7-4410-a154-49e966f1dff6 (dataset), name: DatasetName { pool_name: ZpoolName { id: b478d3fc-de0b-4188-973f-7d9bae6f3554 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 28d3901e-7c34-4f25-9e46-af8d0ab8f21e (dataset): DatasetConfig { id: 28d3901e-7c34-4f25-9e46-af8d0ab8f21e (dataset), name: DatasetName { pool_name: ZpoolName { id: 7dd4c27f-a9f0-4c2e-a0e0-1ad507079428 (zpool), kind: External }, kind: TransientZone { name: \"oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 28f8668a-801d-4618-86a2-1c52105ab565 (dataset): DatasetConfig { id: 28f8668a-801d-4618-86a2-1c52105ab565 (dataset), name: DatasetName { pool_name: ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2c427402-a4a6-485a-b0e8-ad487b2fcb34 (dataset): DatasetConfig { id: 2c427402-a4a6-485a-b0e8-ad487b2fcb34 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_85387c78-aa29-49a7-9f9e-3490ed51f967\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2e9bb792-6540-4648-acb7-808bc465d89c (dataset): DatasetConfig { id: 2e9bb792-6540-4648-acb7-808bc465d89c (dataset), name: DatasetName { pool_name: ZpoolName { id: 7dd4c27f-a9f0-4c2e-a0e0-1ad507079428 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2f7e5109-2165-4ab7-8f7c-b02d367f8705 (dataset): DatasetConfig { id: 2f7e5109-2165-4ab7-8f7c-b02d367f8705 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6 (zpool), kind: External }, kind: TransientZone { name: \"oxz_oximeter_7b35ff06-c97e-4b81-8f7a-0e6c7999e8ed\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3131cb23-868d-429a-883a-1d920e1232a4 (dataset): DatasetConfig { id: 3131cb23-868d-429a-883a-1d920e1232a4 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_e90d0163-99b4-4559-bdb8-a40ce4a2ffa9\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 340a6ad3-147f-4a58-8bad-a6b4dd952360 (dataset): DatasetConfig { id: 340a6ad3-147f-4a58-8bad-a6b4dd952360 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 34fbe576-016a-4fa2-95cf-a5f1d913ca6c (dataset): DatasetConfig { id: 34fbe576-016a-4fa2-95cf-a5f1d913ca6c (dataset), name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3a8f21f5-ef87-43ed-b512-21015ac4a8cf (dataset): DatasetConfig { id: 3a8f21f5-ef87-43ed-b512-21015ac4a8cf (dataset), name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3d7f7b91-431a-4af5-a3f3-196a783c869e (dataset): DatasetConfig { id: 3d7f7b91-431a-4af5-a3f3-196a783c869e (dataset), name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_016d571e-a5ea-4351-b665-3e607b682e72\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 401ff6a9-f4d1-4dca-ab67-80e72f8b4577 (dataset): DatasetConfig { id: 401ff6a9-f4d1-4dca-ab67-80e72f8b4577 (dataset), name: DatasetName { pool_name: ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_780e66c3-354e-48ea-b337-3b7eceac6dc0\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4c720ccb-ae25-4382-a0eb-b2cbe382be06 (dataset): DatasetConfig { id: 4c720ccb-ae25-4382-a0eb-b2cbe382be06 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc2d7f49-c833-4446-a01d-161c755b095b (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4c7f7eca-443b-4990-9ef0-34c08e72b537 (dataset): DatasetConfig { id: 4c7f7eca-443b-4990-9ef0-34c08e72b537 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 530cdfe3-7cd6-4bbb-9365-307690fc6df9 (dataset): DatasetConfig { id: 530cdfe3-7cd6-4bbb-9365-307690fc6df9 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_6b344173-8cba-4e37-b5e5-362444c7655b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 577802f9-8f3a-4f07-8df3-136baab358f5 (dataset): DatasetConfig { id: 577802f9-8f3a-4f07-8df3-136baab358f5 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7dd4c27f-a9f0-4c2e-a0e0-1ad507079428 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 5cbc858d-2d4d-4fb5-a949-9c3c0bbf29df (dataset): DatasetConfig { id: 5cbc858d-2d4d-4fb5-a949-9c3c0bbf29df (dataset), name: DatasetName { pool_name: ZpoolName { id: 6a73c9fc-2f58-450b-97e5-4ef356bb98fb (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 5f29730a-e1bd-4849-b3e9-7f3f6e7b62a1 (dataset): DatasetConfig { id: 5f29730a-e1bd-4849-b3e9-7f3f6e7b62a1 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc2d7f49-c833-4446-a01d-161c755b095b (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5f80f3b8-3a5c-4e2f-bac2-5131cc6d1961 (dataset): DatasetConfig { id: 5f80f3b8-3a5c-4e2f-bac2-5131cc6d1961 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc2d7f49-c833-4446-a01d-161c755b095b (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_f7f17c9a-0027-41cc-b185-223fb703772f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 692ca293-5453-4726-97c9-ab0949f77fad (dataset): DatasetConfig { id: 692ca293-5453-4726-97c9-ab0949f77fad (dataset), name: DatasetName { pool_name: ZpoolName { id: 6a73c9fc-2f58-450b-97e5-4ef356bb98fb (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6b9f94dc-3eb0-4d70-beb2-b69c9252013b (dataset): DatasetConfig { id: 6b9f94dc-3eb0-4d70-beb2-b69c9252013b (dataset), name: DatasetName { pool_name: ZpoolName { id: 7dd4c27f-a9f0-4c2e-a0e0-1ad507079428 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 72ee2981-8311-437c-9462-84cd0aa7894b (dataset): DatasetConfig { id: 72ee2981-8311-437c-9462-84cd0aa7894b (dataset), name: DatasetName { pool_name: ZpoolName { id: 6a73c9fc-2f58-450b-97e5-4ef356bb98fb (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_71d88fcd-cb6a-4be5-9a21-e924eee85be6\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 74456b27-84f1-4405-8925-03d0c3daaf91 (dataset): DatasetConfig { id: 74456b27-84f1-4405-8925-03d0c3daaf91 (dataset), name: DatasetName { pool_name: ZpoolName { id: 6a73c9fc-2f58-450b-97e5-4ef356bb98fb (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 80b0bb64-5003-4140-a3ae-e728de4e6520 (dataset): DatasetConfig { id: 80b0bb64-5003-4140-a3ae-e728de4e6520 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_f1598ca1-8c9f-4830-9931-a5c2d7746097\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8179d1df-ac1b-4591-819d-b435b32912ff (dataset): DatasetConfig { id: 8179d1df-ac1b-4591-819d-b435b32912ff (dataset), name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 8a665284-3bfc-4da5-93a5-5185366ab28f (dataset): DatasetConfig { id: 8a665284-3bfc-4da5-93a5-5185366ab28f (dataset), name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 92000373-7eed-4d74-a43a-17556b1c575a (dataset): DatasetConfig { id: 92000373-7eed-4d74-a43a-17556b1c575a (dataset), name: DatasetName { pool_name: ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9cc0b486-5e37-481a-9056-b816922e4f6b (dataset): DatasetConfig { id: 9cc0b486-5e37-481a-9056-b816922e4f6b (dataset), name: DatasetName { pool_name: ZpoolName { id: bc2d7f49-c833-4446-a01d-161c755b095b (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_d4420bcb-68b6-430d-b904-17ad9aef41ba\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9d06b2cc-fe6c-46b1-8212-855979a47b0d (dataset): DatasetConfig { id: 9d06b2cc-fe6c-46b1-8212-855979a47b0d (dataset), name: DatasetName { pool_name: ZpoolName { id: bc2d7f49-c833-4446-a01d-161c755b095b (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9ee73f03-a978-42f0-b407-b38cb4b8a8ff (dataset): DatasetConfig { id: 9ee73f03-a978-42f0-b407-b38cb4b8a8ff (dataset), name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, a098cdbb-70da-4d36-be68-aa314b750bd0 (dataset): DatasetConfig { id: a098cdbb-70da-4d36-be68-aa314b750bd0 (dataset), name: DatasetName { pool_name: ZpoolName { id: b478d3fc-de0b-4188-973f-7d9bae6f3554 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a180f7ba-3a40-4377-91b8-9a8c85f14c1e (dataset): DatasetConfig { id: a180f7ba-3a40-4377-91b8-9a8c85f14c1e (dataset), name: DatasetName { pool_name: ZpoolName { id: bc2d7f49-c833-4446-a01d-161c755b095b (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, a1ca7c88-ad96-4060-b48b-6a7c079b3bd4 (dataset): DatasetConfig { id: a1ca7c88-ad96-4060-b48b-6a7c079b3bd4 (dataset), name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_93524fa4-4882-458b-a91f-54ec48cb5768\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a253db52-2f3b-4246-bd95-d4d684185c22 (dataset): DatasetConfig { id: a253db52-2f3b-4246-bd95-d4d684185c22 (dataset), name: DatasetName { pool_name: ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a3fd4fdf-f354-4385-910c-df2740ca61f6 (dataset): DatasetConfig { id: a3fd4fdf-f354-4385-910c-df2740ca61f6 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, ac8df8ce-67ef-41a1-b5f6-3fdbcc21e575 (dataset): DatasetConfig { id: ac8df8ce-67ef-41a1-b5f6-3fdbcc21e575 (dataset), name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ae82aeb1-d80c-4b4a-a588-6dbf6b264bd2 (dataset): DatasetConfig { id: ae82aeb1-d80c-4b4a-a588-6dbf6b264bd2 (dataset), name: DatasetName { pool_name: ZpoolName { id: b478d3fc-de0b-4188-973f-7d9bae6f3554 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_e506936b-b474-4691-885c-9f5f291e9b52\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b26b5777-dfb9-4ca3-a0cd-d6629181b00b (dataset): DatasetConfig { id: b26b5777-dfb9-4ca3-a0cd-d6629181b00b (dataset), name: DatasetName { pool_name: ZpoolName { id: 7dd4c27f-a9f0-4c2e-a0e0-1ad507079428 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_fae1a638-2f10-4b23-9417-1d370e042bb5\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b2e23b21-2efb-42cf-b657-13d8a8f6d10e (dataset): DatasetConfig { id: b2e23b21-2efb-42cf-b657-13d8a8f6d10e (dataset), name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, bc8cd983-8247-429a-b32b-9955a13910cd (dataset): DatasetConfig { id: bc8cd983-8247-429a-b32b-9955a13910cd (dataset), name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_e43e6ae4-7893-4170-ad06-c06d9fd9de91\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, be6889a8-2277-426f-b49f-95b22c72944d (dataset): DatasetConfig { id: be6889a8-2277-426f-b49f-95b22c72944d (dataset), name: DatasetName { pool_name: ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, c4b9c223-e09f-4fe1-86ab-44491d862baf (dataset): DatasetConfig { id: c4b9c223-e09f-4fe1-86ab-44491d862baf (dataset), name: DatasetName { pool_name: ZpoolName { id: 7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_c3f170b5-2656-46a9-9552-0acd23b28479\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c5e2b4e8-3414-4550-b8a3-93b1fdc4c76d (dataset): DatasetConfig { id: c5e2b4e8-3414-4550-b8a3-93b1fdc4c76d (dataset), name: DatasetName { pool_name: ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_d993ecda-7c61-4422-b656-98f08a42bf37\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c8669db1-e304-4bd9-8bdb-2fa3899eff78 (dataset): DatasetConfig { id: c8669db1-e304-4bd9-8bdb-2fa3899eff78 (dataset), name: DatasetName { pool_name: ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ce0571b9-b5fb-4253-88ce-f65b4d5f15ea (dataset): DatasetConfig { id: ce0571b9-b5fb-4253-88ce-f65b4d5f15ea (dataset), name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d807188a-ad42-42e8-9d0d-ca352e77599b (dataset): DatasetConfig { id: d807188a-ad42-42e8-9d0d-ca352e77599b (dataset), name: DatasetName { pool_name: ZpoolName { id: 7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d88807bb-10cb-4670-af05-0479a7fb1fbd (dataset): DatasetConfig { id: d88807bb-10cb-4670-af05-0479a7fb1fbd (dataset), name: DatasetName { pool_name: ZpoolName { id: b478d3fc-de0b-4188-973f-7d9bae6f3554 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, db3c74ba-01e4-42e1-a228-e56424323a3a (dataset): DatasetConfig { id: db3c74ba-01e4-42e1-a228-e56424323a3a (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, decdff62-64c0-48d0-97bd-4fad4c5a654b (dataset): DatasetConfig { id: decdff62-64c0-48d0-97bd-4fad4c5a654b (dataset), name: DatasetName { pool_name: ZpoolName { id: 7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_4f2d90dc-0f1a-4fef-a6eb-7e9430c6f5cb\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e70ebeda-90a5-47ad-9548-1ec6610a5901 (dataset): DatasetConfig { id: e70ebeda-90a5-47ad-9548-1ec6610a5901 (dataset), name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e723210d-fa11-4093-b06d-08bb368024fd (dataset): DatasetConfig { id: e723210d-fa11-4093-b06d-08bb368024fd (dataset), name: DatasetName { pool_name: ZpoolName { id: b478d3fc-de0b-4188-973f-7d9bae6f3554 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_407c6397-e5e8-474a-82a5-2ddcd6a5fbb1\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e7e00d8c-a667-443a-a692-9215971872cb (dataset): DatasetConfig { id: e7e00d8c-a667-443a-a692-9215971872cb (dataset), name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, eca1f5d3-ab9d-4603-9252-d98bd759fc44 (dataset): DatasetConfig { id: eca1f5d3-ab9d-4603-9252-d98bd759fc44 (dataset), name: DatasetName { pool_name: ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_80a37bc6-efc5-46e2-8f04-a801f379f3c2\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ed7184cd-4031-4833-b0b1-927a52e3f86b (dataset): DatasetConfig { id: ed7184cd-4031-4833-b0b1-927a52e3f86b (dataset), name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f7d057c6-023b-4c28-a9fa-b41942b56c16 (dataset): DatasetConfig { id: f7d057c6-023b-4c28-a9fa-b41942b56c16 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f955c46d-6b26-42f6-bcb5-4d98ba67fe79 (dataset): DatasetConfig { id: f955c46d-6b26-42f6-bcb5-4d98ba67fe79 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fb67d044-b12d-4703-99d5-4e2b2ebfb5f8 (dataset): DatasetConfig { id: fb67d044-b12d-4703-99d5-4e2b2ebfb5f8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fee779be-7f25-4ef4-8636-422becca3d95 (dataset): DatasetConfig { id: fee779be-7f25-4ef4-8636-422becca3d95 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }} }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.805415111Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/651ad8e9-444b-45dc-9dfc-4c79c25fd16e/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.805422621Z","hostname":"buskin","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/4a331860-fd72-42e5-801b-f6e26734b6a6/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.805427321Z","hostname":"buskin","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested datasets', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.805431221Z","hostname":"buskin","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:909"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.822105841Z","hostname":"buskin","pid":1566,"name":"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/zone/oxz_clickhouse_4d4b2424-58da-49f4-8f26-04c0d70d01b3","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.822144752Z","hostname":"buskin","pid":1566,"name":"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/zone/oxz_clickhouse_4d4b2424-58da-49f4-8f26-04c0d70d01b3","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.822151512Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/zone/oxz_clickhouse_4d4b2424-58da-49f4-8f26-04c0d70d01b3\"), full_name: \"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/zone/oxz_clickhouse_4d4b2424-58da-49f4-8f26-04c0d70d01b3\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.902596665Z","hostname":"buskin","pid":1566,"name":"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/zone/oxz_crucible_1c81cbcf-1fc4-4d48-a274-adee5ad7eaa2","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.902635046Z","hostname":"buskin","pid":1566,"name":"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/zone/oxz_crucible_1c81cbcf-1fc4-4d48-a274-adee5ad7eaa2","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.902639646Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/zone/oxz_crucible_1c81cbcf-1fc4-4d48-a274-adee5ad7eaa2\"), full_name: \"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/zone/oxz_crucible_1c81cbcf-1fc4-4d48-a274-adee5ad7eaa2\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.966865639Z","hostname":"buskin","pid":1566,"name":"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.96691843Z","hostname":"buskin","pid":1566,"name":"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:39.96692454Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.032705909Z","hostname":"buskin","pid":1566,"name":"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.03275721Z","hostname":"buskin","pid":1566,"name":"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.03276305Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/external_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.09819419Z","hostname":"buskin","pid":1566,"name":"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/zone/oxz_external_dns_754b7158-e16d-4a6a-a0bc-b68f75a4910b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.098242061Z","hostname":"buskin","pid":1566,"name":"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/zone/oxz_external_dns_754b7158-e16d-4a6a-a0bc-b68f75a4910b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.098247611Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/zone/oxz_external_dns_754b7158-e16d-4a6a-a0bc-b68f75a4910b\"), full_name: \"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/zone/oxz_external_dns_754b7158-e16d-4a6a-a0bc-b68f75a4910b\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.162607488Z","hostname":"buskin","pid":1566,"name":"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crypt/zone/oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.162654989Z","hostname":"buskin","pid":1566,"name":"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crypt/zone/oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.162660529Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f628b386-82f1-43d6-bae7-1b54d721271c/crypt/zone/oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6\"), full_name: \"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crypt/zone/oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.233876251Z","hostname":"buskin","pid":1566,"name":"oxp_3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4/crypt/zone/oxz_crucible_b87d7da8-d424-414d-8ec7-c372ad128c96","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.233917022Z","hostname":"buskin","pid":1566,"name":"oxp_3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4/crypt/zone/oxz_crucible_b87d7da8-d424-414d-8ec7-c372ad128c96","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.233922072Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4/crypt/zone/oxz_crucible_b87d7da8-d424-414d-8ec7-c372ad128c96\"), full_name: \"oxp_3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4/crypt/zone/oxz_crucible_b87d7da8-d424-414d-8ec7-c372ad128c96\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.314011406Z","hostname":"buskin","pid":1566,"name":"oxp_b478d3fc-de0b-4188-973f-7d9bae6f3554/crypt/zone/oxz_crucible_pantry_f583c1f5-9142-4c11-9241-e867fa84cb28","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.314052307Z","hostname":"buskin","pid":1566,"name":"oxp_b478d3fc-de0b-4188-973f-7d9bae6f3554/crypt/zone/oxz_crucible_pantry_f583c1f5-9142-4c11-9241-e867fa84cb28","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.314057287Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b478d3fc-de0b-4188-973f-7d9bae6f3554/crypt/zone/oxz_crucible_pantry_f583c1f5-9142-4c11-9241-e867fa84cb28\"), full_name: \"oxp_b478d3fc-de0b-4188-973f-7d9bae6f3554/crypt/zone/oxz_crucible_pantry_f583c1f5-9142-4c11-9241-e867fa84cb28\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.388469313Z","hostname":"buskin","pid":1566,"name":"oxp_b478d3fc-de0b-4188-973f-7d9bae6f3554/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.388504243Z","hostname":"buskin","pid":1566,"name":"oxp_b478d3fc-de0b-4188-973f-7d9bae6f3554/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.388508293Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_b478d3fc-de0b-4188-973f-7d9bae6f3554/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.454319182Z","hostname":"buskin","pid":1566,"name":"oxp_7dd4c27f-a9f0-4c2e-a0e0-1ad507079428/crypt/zone/oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.454362054Z","hostname":"buskin","pid":1566,"name":"oxp_7dd4c27f-a9f0-4c2e-a0e0-1ad507079428/crypt/zone/oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.454366934Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7dd4c27f-a9f0-4c2e-a0e0-1ad507079428/crypt/zone/oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84\"), full_name: \"oxp_7dd4c27f-a9f0-4c2e-a0e0-1ad507079428/crypt/zone/oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.525130746Z","hostname":"buskin","pid":1566,"name":"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.525169716Z","hostname":"buskin","pid":1566,"name":"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.525174136Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f628b386-82f1-43d6-bae7-1b54d721271c/crypt/zone\"), full_name: \"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/46437054-bf3d-4063-a6ee-f11c748dcba3)","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:54:40.536878553Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:238"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.562565568Z","hostname":"buskin","pid":1566,"name":"oxp_3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4/crypt/zone/oxz_cockroachdb_85387c78-aa29-49a7-9f9e-3490ed51f967","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.562583679Z","hostname":"buskin","pid":1566,"name":"oxp_3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4/crypt/zone/oxz_cockroachdb_85387c78-aa29-49a7-9f9e-3490ed51f967","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.562589179Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4/crypt/zone/oxz_cockroachdb_85387c78-aa29-49a7-9f9e-3490ed51f967\"), full_name: \"oxp_3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4/crypt/zone/oxz_cockroachdb_85387c78-aa29-49a7-9f9e-3490ed51f967\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.636992304Z","hostname":"buskin","pid":1566,"name":"oxp_7dd4c27f-a9f0-4c2e-a0e0-1ad507079428/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.637037265Z","hostname":"buskin","pid":1566,"name":"oxp_7dd4c27f-a9f0-4c2e-a0e0-1ad507079428/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.637042205Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7dd4c27f-a9f0-4c2e-a0e0-1ad507079428/crypt/zone\"), full_name: \"oxp_7dd4c27f-a9f0-4c2e-a0e0-1ad507079428/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.6767778Z","hostname":"buskin","pid":1566,"name":"oxp_7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crypt/zone/oxz_oximeter_7b35ff06-c97e-4b81-8f7a-0e6c7999e8ed","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.676814431Z","hostname":"buskin","pid":1566,"name":"oxp_7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crypt/zone/oxz_oximeter_7b35ff06-c97e-4b81-8f7a-0e6c7999e8ed","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.676819241Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crypt/zone/oxz_oximeter_7b35ff06-c97e-4b81-8f7a-0e6c7999e8ed\"), full_name: \"oxp_7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crypt/zone/oxz_oximeter_7b35ff06-c97e-4b81-8f7a-0e6c7999e8ed\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.749961217Z","hostname":"buskin","pid":1566,"name":"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/zone/oxz_cockroachdb_e90d0163-99b4-4559-bdb8-a40ce4a2ffa9","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.750002988Z","hostname":"buskin","pid":1566,"name":"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/zone/oxz_cockroachdb_e90d0163-99b4-4559-bdb8-a40ce4a2ffa9","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.750008328Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/zone/oxz_cockroachdb_e90d0163-99b4-4559-bdb8-a40ce4a2ffa9\"), full_name: \"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/zone/oxz_cockroachdb_e90d0163-99b4-4559-bdb8-a40ce4a2ffa9\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.826858339Z","hostname":"buskin","pid":1566,"name":"oxp_3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.826894749Z","hostname":"buskin","pid":1566,"name":"oxp_3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.82690519Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4/crypt/debug\"), full_name: \"oxp_3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.864147078Z","hostname":"buskin","pid":1566,"name":"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/zone/oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.864194049Z","hostname":"buskin","pid":1566,"name":"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/zone/oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.864198899Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/zone/oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c\"), full_name: \"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/zone/oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.942943803Z","hostname":"buskin","pid":1566,"name":"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.942983144Z","hostname":"buskin","pid":1566,"name":"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.942988694Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/zone\"), full_name: \"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.981874279Z","hostname":"buskin","pid":1566,"name":"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/zone/oxz_cockroachdb_016d571e-a5ea-4351-b665-3e607b682e72","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.98191066Z","hostname":"buskin","pid":1566,"name":"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/zone/oxz_cockroachdb_016d571e-a5ea-4351-b665-3e607b682e72","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:40.98191574Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/zone/oxz_cockroachdb_016d571e-a5ea-4351-b665-3e607b682e72\"), full_name: \"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/zone/oxz_cockroachdb_016d571e-a5ea-4351-b665-3e607b682e72\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.058599617Z","hostname":"buskin","pid":1566,"name":"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crypt/zone/oxz_cockroachdb_780e66c3-354e-48ea-b337-3b7eceac6dc0","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.058646778Z","hostname":"buskin","pid":1566,"name":"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crypt/zone/oxz_cockroachdb_780e66c3-354e-48ea-b337-3b7eceac6dc0","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.058651799Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f628b386-82f1-43d6-bae7-1b54d721271c/crypt/zone/oxz_cockroachdb_780e66c3-354e-48ea-b337-3b7eceac6dc0\"), full_name: \"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crypt/zone/oxz_cockroachdb_780e66c3-354e-48ea-b337-3b7eceac6dc0\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.136619515Z","hostname":"buskin","pid":1566,"name":"oxp_bc2d7f49-c833-4446-a01d-161c755b095b/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.136700166Z","hostname":"buskin","pid":1566,"name":"oxp_bc2d7f49-c833-4446-a01d-161c755b095b/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.136709477Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_bc2d7f49-c833-4446-a01d-161c755b095b/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.199879916Z","hostname":"buskin","pid":1566,"name":"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.199917386Z","hostname":"buskin","pid":1566,"name":"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.199922166Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.262472581Z","hostname":"buskin","pid":1566,"name":"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/zone/oxz_external_dns_6b344173-8cba-4e37-b5e5-362444c7655b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.262510592Z","hostname":"buskin","pid":1566,"name":"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/zone/oxz_external_dns_6b344173-8cba-4e37-b5e5-362444c7655b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.262521392Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/zone/oxz_external_dns_6b344173-8cba-4e37-b5e5-362444c7655b\"), full_name: \"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/zone/oxz_external_dns_6b344173-8cba-4e37-b5e5-362444c7655b\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.334758108Z","hostname":"buskin","pid":1566,"name":"oxp_7dd4c27f-a9f0-4c2e-a0e0-1ad507079428/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.334796569Z","hostname":"buskin","pid":1566,"name":"oxp_7dd4c27f-a9f0-4c2e-a0e0-1ad507079428/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.334801429Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7dd4c27f-a9f0-4c2e-a0e0-1ad507079428/crypt/debug\"), full_name: \"oxp_7dd4c27f-a9f0-4c2e-a0e0-1ad507079428/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.371534806Z","hostname":"buskin","pid":1566,"name":"oxp_6a73c9fc-2f58-450b-97e5-4ef356bb98fb/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.371568227Z","hostname":"buskin","pid":1566,"name":"oxp_6a73c9fc-2f58-450b-97e5-4ef356bb98fb/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.371572936Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/6a73c9fc-2f58-450b-97e5-4ef356bb98fb/crypt/debug\"), full_name: \"oxp_6a73c9fc-2f58-450b-97e5-4ef356bb98fb/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.408355885Z","hostname":"buskin","pid":1566,"name":"oxp_bc2d7f49-c833-4446-a01d-161c755b095b/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.408388465Z","hostname":"buskin","pid":1566,"name":"oxp_bc2d7f49-c833-4446-a01d-161c755b095b/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.408393045Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/bc2d7f49-c833-4446-a01d-161c755b095b/crypt/zone\"), full_name: \"oxp_bc2d7f49-c833-4446-a01d-161c755b095b/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.445282576Z","hostname":"buskin","pid":1566,"name":"oxp_bc2d7f49-c833-4446-a01d-161c755b095b/crypt/zone/oxz_crucible_f7f17c9a-0027-41cc-b185-223fb703772f","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.445315756Z","hostname":"buskin","pid":1566,"name":"oxp_bc2d7f49-c833-4446-a01d-161c755b095b/crypt/zone/oxz_crucible_f7f17c9a-0027-41cc-b185-223fb703772f","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.445320376Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/bc2d7f49-c833-4446-a01d-161c755b095b/crypt/zone/oxz_crucible_f7f17c9a-0027-41cc-b185-223fb703772f\"), full_name: \"oxp_bc2d7f49-c833-4446-a01d-161c755b095b/crypt/zone/oxz_crucible_f7f17c9a-0027-41cc-b185-223fb703772f\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.516840706Z","hostname":"buskin","pid":1566,"name":"oxp_6a73c9fc-2f58-450b-97e5-4ef356bb98fb/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.516882097Z","hostname":"buskin","pid":1566,"name":"oxp_6a73c9fc-2f58-450b-97e5-4ef356bb98fb/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.516887917Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/6a73c9fc-2f58-450b-97e5-4ef356bb98fb/crypt/zone\"), full_name: \"oxp_6a73c9fc-2f58-450b-97e5-4ef356bb98fb/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/46437054-bf3d-4063-a6ee-f11c748dcba3)","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:54:41.538961829Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:238"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.555075207Z","hostname":"buskin","pid":1566,"name":"oxp_7dd4c27f-a9f0-4c2e-a0e0-1ad507079428/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.555085427Z","hostname":"buskin","pid":1566,"name":"oxp_7dd4c27f-a9f0-4c2e-a0e0-1ad507079428/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.555089797Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_7dd4c27f-a9f0-4c2e-a0e0-1ad507079428/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.617742174Z","hostname":"buskin","pid":1566,"name":"oxp_6a73c9fc-2f58-450b-97e5-4ef356bb98fb/crypt/zone/oxz_crucible_71d88fcd-cb6a-4be5-9a21-e924eee85be6","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.617781505Z","hostname":"buskin","pid":1566,"name":"oxp_6a73c9fc-2f58-450b-97e5-4ef356bb98fb/crypt/zone/oxz_crucible_71d88fcd-cb6a-4be5-9a21-e924eee85be6","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.617786045Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/6a73c9fc-2f58-450b-97e5-4ef356bb98fb/crypt/zone/oxz_crucible_71d88fcd-cb6a-4be5-9a21-e924eee85be6\"), full_name: \"oxp_6a73c9fc-2f58-450b-97e5-4ef356bb98fb/crypt/zone/oxz_crucible_71d88fcd-cb6a-4be5-9a21-e924eee85be6\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.688118857Z","hostname":"buskin","pid":1566,"name":"oxp_6a73c9fc-2f58-450b-97e5-4ef356bb98fb/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.688162318Z","hostname":"buskin","pid":1566,"name":"oxp_6a73c9fc-2f58-450b-97e5-4ef356bb98fb/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.688168138Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_6a73c9fc-2f58-450b-97e5-4ef356bb98fb/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.753286292Z","hostname":"buskin","pid":1566,"name":"oxp_7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crypt/zone/oxz_nexus_f1598ca1-8c9f-4830-9931-a5c2d7746097","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.753331703Z","hostname":"buskin","pid":1566,"name":"oxp_7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crypt/zone/oxz_nexus_f1598ca1-8c9f-4830-9931-a5c2d7746097","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.753336113Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crypt/zone/oxz_nexus_f1598ca1-8c9f-4830-9931-a5c2d7746097\"), full_name: \"oxp_7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crypt/zone/oxz_nexus_f1598ca1-8c9f-4830-9931-a5c2d7746097\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.824376741Z","hostname":"buskin","pid":1566,"name":"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.824418432Z","hostname":"buskin","pid":1566,"name":"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.824434232Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/debug\"), full_name: \"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.860664788Z","hostname":"buskin","pid":1566,"name":"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.860701369Z","hostname":"buskin","pid":1566,"name":"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.860705349Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.923645732Z","hostname":"buskin","pid":1566,"name":"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.923678883Z","hostname":"buskin","pid":1566,"name":"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.923682883Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.989334159Z","hostname":"buskin","pid":1566,"name":"oxp_bc2d7f49-c833-4446-a01d-161c755b095b/crypt/zone/oxz_cockroachdb_d4420bcb-68b6-430d-b904-17ad9aef41ba","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.98937812Z","hostname":"buskin","pid":1566,"name":"oxp_bc2d7f49-c833-4446-a01d-161c755b095b/crypt/zone/oxz_cockroachdb_d4420bcb-68b6-430d-b904-17ad9aef41ba","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:41.989383469Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/bc2d7f49-c833-4446-a01d-161c755b095b/crypt/zone/oxz_cockroachdb_d4420bcb-68b6-430d-b904-17ad9aef41ba\"), full_name: \"oxp_bc2d7f49-c833-4446-a01d-161c755b095b/crypt/zone/oxz_cockroachdb_d4420bcb-68b6-430d-b904-17ad9aef41ba\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.065662387Z","hostname":"buskin","pid":1566,"name":"oxp_bc2d7f49-c833-4446-a01d-161c755b095b/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.065743279Z","hostname":"buskin","pid":1566,"name":"oxp_bc2d7f49-c833-4446-a01d-161c755b095b/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.065752489Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_bc2d7f49-c833-4446-a01d-161c755b095b/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.126492683Z","hostname":"buskin","pid":1566,"name":"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.126526824Z","hostname":"buskin","pid":1566,"name":"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.126530864Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/debug\"), full_name: \"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.163392043Z","hostname":"buskin","pid":1566,"name":"oxp_b478d3fc-de0b-4188-973f-7d9bae6f3554/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.163428204Z","hostname":"buskin","pid":1566,"name":"oxp_b478d3fc-de0b-4188-973f-7d9bae6f3554/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.163432494Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b478d3fc-de0b-4188-973f-7d9bae6f3554/crypt/zone\"), full_name: \"oxp_b478d3fc-de0b-4188-973f-7d9bae6f3554/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.201914871Z","hostname":"buskin","pid":1566,"name":"oxp_bc2d7f49-c833-4446-a01d-161c755b095b/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.201946762Z","hostname":"buskin","pid":1566,"name":"oxp_bc2d7f49-c833-4446-a01d-161c755b095b/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.201950702Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/bc2d7f49-c833-4446-a01d-161c755b095b/crypt/debug\"), full_name: \"oxp_bc2d7f49-c833-4446-a01d-161c755b095b/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.238472254Z","hostname":"buskin","pid":1566,"name":"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/zone/oxz_crucible_93524fa4-4882-458b-a91f-54ec48cb5768","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.238507655Z","hostname":"buskin","pid":1566,"name":"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/zone/oxz_crucible_93524fa4-4882-458b-a91f-54ec48cb5768","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.238511884Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/zone/oxz_crucible_93524fa4-4882-458b-a91f-54ec48cb5768\"), full_name: \"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/zone/oxz_crucible_93524fa4-4882-458b-a91f-54ec48cb5768\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.311009326Z","hostname":"buskin","pid":1566,"name":"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.311045157Z","hostname":"buskin","pid":1566,"name":"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.311050347Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.375027914Z","hostname":"buskin","pid":1566,"name":"oxp_7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.375063765Z","hostname":"buskin","pid":1566,"name":"oxp_7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.375067765Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crypt/debug\"), full_name: \"oxp_7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.411535036Z","hostname":"buskin","pid":1566,"name":"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.411567137Z","hostname":"buskin","pid":1566,"name":"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.411577307Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/external_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.474528371Z","hostname":"buskin","pid":1566,"name":"oxp_b478d3fc-de0b-4188-973f-7d9bae6f3554/crypt/zone/oxz_crucible_e506936b-b474-4691-885c-9f5f291e9b52","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.474564982Z","hostname":"buskin","pid":1566,"name":"oxp_b478d3fc-de0b-4188-973f-7d9bae6f3554/crypt/zone/oxz_crucible_e506936b-b474-4691-885c-9f5f291e9b52","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.474569092Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b478d3fc-de0b-4188-973f-7d9bae6f3554/crypt/zone/oxz_crucible_e506936b-b474-4691-885c-9f5f291e9b52\"), full_name: \"oxp_b478d3fc-de0b-4188-973f-7d9bae6f3554/crypt/zone/oxz_crucible_e506936b-b474-4691-885c-9f5f291e9b52\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/46437054-bf3d-4063-a6ee-f11c748dcba3)","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:54:42.540551635Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:238"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.546781227Z","hostname":"buskin","pid":1566,"name":"oxp_7dd4c27f-a9f0-4c2e-a0e0-1ad507079428/crypt/zone/oxz_crucible_fae1a638-2f10-4b23-9417-1d370e042bb5","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.546787967Z","hostname":"buskin","pid":1566,"name":"oxp_7dd4c27f-a9f0-4c2e-a0e0-1ad507079428/crypt/zone/oxz_crucible_fae1a638-2f10-4b23-9417-1d370e042bb5","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.546791907Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7dd4c27f-a9f0-4c2e-a0e0-1ad507079428/crypt/zone/oxz_crucible_fae1a638-2f10-4b23-9417-1d370e042bb5\"), full_name: \"oxp_7dd4c27f-a9f0-4c2e-a0e0-1ad507079428/crypt/zone/oxz_crucible_fae1a638-2f10-4b23-9417-1d370e042bb5\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.618711675Z","hostname":"buskin","pid":1566,"name":"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/clickhouse","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.618749196Z","hostname":"buskin","pid":1566,"name":"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/clickhouse","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.618764457Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/clickhouse\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.683514551Z","hostname":"buskin","pid":1566,"name":"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/zone/oxz_crucible_pantry_e43e6ae4-7893-4170-ad06-c06d9fd9de91","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.683549832Z","hostname":"buskin","pid":1566,"name":"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/zone/oxz_crucible_pantry_e43e6ae4-7893-4170-ad06-c06d9fd9de91","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.683553802Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/zone/oxz_crucible_pantry_e43e6ae4-7893-4170-ad06-c06d9fd9de91\"), full_name: \"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/zone/oxz_crucible_pantry_e43e6ae4-7893-4170-ad06-c06d9fd9de91\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.757284362Z","hostname":"buskin","pid":1566,"name":"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.757328033Z","hostname":"buskin","pid":1566,"name":"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.757332533Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f628b386-82f1-43d6-bae7-1b54d721271c/crypt/debug\"), full_name: \"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.793684211Z","hostname":"buskin","pid":1566,"name":"oxp_7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crypt/zone/oxz_crucible_c3f170b5-2656-46a9-9552-0acd23b28479","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.793720222Z","hostname":"buskin","pid":1566,"name":"oxp_7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crypt/zone/oxz_crucible_c3f170b5-2656-46a9-9552-0acd23b28479","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.793724942Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crypt/zone/oxz_crucible_c3f170b5-2656-46a9-9552-0acd23b28479\"), full_name: \"oxp_7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crypt/zone/oxz_crucible_c3f170b5-2656-46a9-9552-0acd23b28479\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.870493881Z","hostname":"buskin","pid":1566,"name":"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crypt/zone/oxz_crucible_pantry_d993ecda-7c61-4422-b656-98f08a42bf37","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.870529632Z","hostname":"buskin","pid":1566,"name":"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crypt/zone/oxz_crucible_pantry_d993ecda-7c61-4422-b656-98f08a42bf37","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.870533912Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f628b386-82f1-43d6-bae7-1b54d721271c/crypt/zone/oxz_crucible_pantry_d993ecda-7c61-4422-b656-98f08a42bf37\"), full_name: \"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crypt/zone/oxz_crucible_pantry_d993ecda-7c61-4422-b656-98f08a42bf37\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.941119149Z","hostname":"buskin","pid":1566,"name":"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.94115905Z","hostname":"buskin","pid":1566,"name":"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:42.94116457Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.007052881Z","hostname":"buskin","pid":1566,"name":"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/zone/oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.007100372Z","hostname":"buskin","pid":1566,"name":"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/zone/oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.007104902Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/zone/oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09\"), full_name: \"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crypt/zone/oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.084614468Z","hostname":"buskin","pid":1566,"name":"oxp_7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.084669999Z","hostname":"buskin","pid":1566,"name":"oxp_7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.084674659Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.152889293Z","hostname":"buskin","pid":1566,"name":"oxp_b478d3fc-de0b-4188-973f-7d9bae6f3554/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.152926334Z","hostname":"buskin","pid":1566,"name":"oxp_b478d3fc-de0b-4188-973f-7d9bae6f3554/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.152931574Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b478d3fc-de0b-4188-973f-7d9bae6f3554/crypt/debug\"), full_name: \"oxp_b478d3fc-de0b-4188-973f-7d9bae6f3554/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.192517036Z","hostname":"buskin","pid":1566,"name":"oxp_3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.192572667Z","hostname":"buskin","pid":1566,"name":"oxp_3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.192581297Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.258225372Z","hostname":"buskin","pid":1566,"name":"oxp_7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crypt/zone/oxz_nexus_4f2d90dc-0f1a-4fef-a6eb-7e9430c6f5cb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.258259703Z","hostname":"buskin","pid":1566,"name":"oxp_7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crypt/zone/oxz_nexus_4f2d90dc-0f1a-4fef-a6eb-7e9430c6f5cb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.258264364Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crypt/zone/oxz_nexus_4f2d90dc-0f1a-4fef-a6eb-7e9430c6f5cb\"), full_name: \"oxp_7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crypt/zone/oxz_nexus_4f2d90dc-0f1a-4fef-a6eb-7e9430c6f5cb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.330604451Z","hostname":"buskin","pid":1566,"name":"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.330641562Z","hostname":"buskin","pid":1566,"name":"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.330646312Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.393791501Z","hostname":"buskin","pid":1566,"name":"oxp_b478d3fc-de0b-4188-973f-7d9bae6f3554/crypt/zone/oxz_nexus_407c6397-e5e8-474a-82a5-2ddcd6a5fbb1","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.393822211Z","hostname":"buskin","pid":1566,"name":"oxp_b478d3fc-de0b-4188-973f-7d9bae6f3554/crypt/zone/oxz_nexus_407c6397-e5e8-474a-82a5-2ddcd6a5fbb1","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.393826192Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b478d3fc-de0b-4188-973f-7d9bae6f3554/crypt/zone/oxz_nexus_407c6397-e5e8-474a-82a5-2ddcd6a5fbb1\"), full_name: \"oxp_b478d3fc-de0b-4188-973f-7d9bae6f3554/crypt/zone/oxz_nexus_407c6397-e5e8-474a-82a5-2ddcd6a5fbb1\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.465940014Z","hostname":"buskin","pid":1566,"name":"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.465978855Z","hostname":"buskin","pid":1566,"name":"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.465983885Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.530597777Z","hostname":"buskin","pid":1566,"name":"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crypt/zone/oxz_crucible_80a37bc6-efc5-46e2-8f04-a801f379f3c2","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.530629048Z","hostname":"buskin","pid":1566,"name":"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crypt/zone/oxz_crucible_80a37bc6-efc5-46e2-8f04-a801f379f3c2","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.530643218Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f628b386-82f1-43d6-bae7-1b54d721271c/crypt/zone/oxz_crucible_80a37bc6-efc5-46e2-8f04-a801f379f3c2\"), full_name: \"oxp_f628b386-82f1-43d6-bae7-1b54d721271c/crypt/zone/oxz_crucible_80a37bc6-efc5-46e2-8f04-a801f379f3c2\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/46437054-bf3d-4063-a6ee-f11c748dcba3)","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:54:43.542681822Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:238"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:43.59484632Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"15.123407556s"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.601744128Z","hostname":"buskin","pid":1566,"name":"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.601760678Z","hostname":"buskin","pid":1566,"name":"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.601766888Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/zone\"), full_name: \"oxp_a595bf70-cbb4-4ef8-8714-4d4d03d8ec49/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.638695469Z","hostname":"buskin","pid":1566,"name":"oxp_3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.63872756Z","hostname":"buskin","pid":1566,"name":"oxp_3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.6387315Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.700566549Z","hostname":"buskin","pid":1566,"name":"oxp_3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.70060309Z","hostname":"buskin","pid":1566,"name":"oxp_3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.70061185Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4/crypt/zone\"), full_name: \"oxp_3b2b6eb7-cd9b-441a-aa3a-093f4262f4d4/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.73747148Z","hostname":"buskin","pid":1566,"name":"oxp_7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.73750441Z","hostname":"buskin","pid":1566,"name":"oxp_7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.7375082Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crypt/zone\"), full_name: \"oxp_7a205aaa-ce80-4ec5-b0fd-cc49d61cf4a6/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.77525303Z","hostname":"buskin","pid":1566,"name":"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.775285301Z","hostname":"buskin","pid":1566,"name":"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.775289501Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_1f1780df-c4f8-4ea2-9680-811efa3a608f/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Writing ledger to /pool/int/651ad8e9-444b-45dc-9dfc-4c79c25fd16e/config/.omicron-datasets.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.839821761Z","hostname":"buskin","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/4a331860-fd72-42e5-801b-f6e26734b6a6/config/.omicron-datasets.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.840192129Z","hostname":"buskin","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"datasets ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.840493536Z","hostname":"buskin","pid":1566,"sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:863"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.840610999Z","hostname":"buskin","pid":1566,"uri":"/datasets","method":"PUT","req_id":"3e86bd36-58de-44ba-8d4b-1da2fef7fe9b","remote_addr":"[fd00:1122:3344:101::1]:36955","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":4035905,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 4ffccf6c-3438-4d74-b934-174aa54ef45c (service), filesystem_pool: Some(ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 }, image_source: InstallDataset }, OmicronZoneConfig { id: 1cb0bf88-9c23-4d9e-90d4-741665f48d09 (service), filesystem_pool: Some(ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 }, image_source: InstallDataset }, OmicronZoneConfig { id: 92ca8da9-bfa1-4f67-902e-4f82e77821d6 (service), filesystem_pool: Some(ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 }, image_source: InstallDataset }] }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.87271119Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.872879264Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:38002"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.873087259Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.873094259Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.884005727Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.884039028Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.893300879Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.89334029Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.902272263Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/651ad8e9-444b-45dc-9dfc-4c79c25fd16e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.902312204Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/4a331860-fd72-42e5-801b-f6e26734b6a6/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.902332965Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"ensure_all_omicron_zones: request reconciliation done","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.902414327Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4918","num_zones_updated":0,"num_zones_to_be_added":3,"num_zones_to_be_removed":0} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.902422767Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.945747184Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.945779745Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.945785195Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:43.998226129Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:44.002137658Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:44.006060417Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:44.018647514Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:44.023237098Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:44.02682382Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/46437054-bf3d-4063-a6ee-f11c748dcba3)","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:54:44.544930352Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:238"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:45.595091034Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4363"} {"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:45.595150716Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4373","gateway":"fd00:1122:3344:101::1"} {"msg":"Default route already exists","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:45.600697502Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4387","gateway":"fd00:1122:3344:101::1"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:45.600729183Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4452"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:45.742671586Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4493"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:45.742713667Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4520"} {"msg":"Profile for oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:45.763672334Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:45.787286712Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4533"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:45.787316293Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4404"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:45.834409256Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4446"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:45.834449947Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4500","rack_subnet":"fd00:1122:3344::"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:45.853247365Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4511"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:45.853282485Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4584"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:45.91975364Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4616"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:45.91978648Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4622"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.009109845Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4663"} {"msg":"configuring tfport service","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.009160876Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4536"} {"msg":"refreshed tfport service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.100284352Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4569"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.100329663Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:370"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/46437054-bf3d-4063-a6ee-f11c748dcba3)","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:46.133043328Z","hostname":"buskin","pid":1566,"component":"NexusNotifierTask","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"sled-agent/src/nexus.rs:530"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.133480208Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:420"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:54:46.13490142Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13828796589641439241"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:54:46.135035053Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17564117138543498946"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.164978145Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:717"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.165091248Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:762"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.165534528Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:765","uptime":"1"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.165550448Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:444","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: Some(\n None,\n ),\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n tx_eq: None,\n },\n },\n },\n}"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.202100801Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4190"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.24638356Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.246479642Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.253624905Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.253673916Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.253685106Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.253704146Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.253712787Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.253721137Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.253730077Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.253737877Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.253744567Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.253752518Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.253759088Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.253766858Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.253777278Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.253783488Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.253788348Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.253793099Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.253807559Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.253812399Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.253816969Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.253821809Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.253826919Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.253831559Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.25383598Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:54:46.3249807Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:54:46.325103123Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.72622037Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.726285751Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.726437885Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Booting oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6 zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.726475266Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:489"} {"msg":"Booting oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09 zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.731346137Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:489"} {"msg":"Booting oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:46.735126343Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:489"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c\") failed: Property not found. retry in 41.91369ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:46.857360307Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6\") failed: Property not found. retry in 32.925092ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:46.86408789Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09\") failed: Property not found. retry in 31.500773ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:46.870376484Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6\") failed: Property not found. retry in 97.908772ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:46.904145613Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09\") failed: Property not found. retry in 137.988638ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:46.910350284Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c\") failed: Property not found. retry in 75.945689ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:46.916329111Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c\") failed: Property not found. retry in 221.778129ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:47.043062317Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6\") failed: Property not found. retry in 194.111186ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:47.197653859Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09\") failed: Property not found. retry in 299.184199ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:47.265067144Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c\") failed: Property not found. retry in 365.569337ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:47.278285955Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6\") failed: Property not found. retry in 206.048127ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:47.43606937Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09\") failed: Property not found. retry in 415.974339ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:47.591844508Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6\") failed: Property not found. retry in 1.122215408s","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:47.652916419Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c\") failed: Property not found. retry in 666.099361ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:47.663772097Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09\") failed: Property not found. retry in 833.597498ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:48.018476066Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c\") failed: Property not found. retry in 909.915762ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:48.339959909Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6\") failed: Property not found. retry in 1.183345762s","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:48.784874404Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09\") failed: Property not found. retry in 1.48930362s","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:48.861885289Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c\") failed: Property not found. retry in 676.474296ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:49.261210904Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c\") failed: Property not found. retry in 1.065617523s","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:49.946613057Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6\") failed: Property not found. retry in 1.294976064s","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:49.977202754Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09\") failed: Property not found. retry in 1.168590446s","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:50.359988994Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c\") failed: Property not found. retry in 1.085663755s","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:51.021353959Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6\") failed: Property not found. retry in 663.674914ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:51.282040997Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09\") failed: Property not found. retry in 1.292215326s","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:51.537801983Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6\") failed: Property not found. retry in 855.793252ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:51.955648931Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c\") failed: Property not found. retry in 928.028805ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:52.116845533Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.065013971Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3377","zone":"oxz_internal_dns_92ca8da9-bfa1-4f67-902e-4f82e77821d6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.065061902Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3377","zone":"oxz_internal_dns_1cb0bf88-9c23-4d9e-90d4-741665f48d09"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.065066973Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3377","zone":"oxz_internal_dns_4ffccf6c-3438-4d74-b934-174aa54ef45c"} {"msg":"Writing ledger to /pool/int/651ad8e9-444b-45dc-9dfc-4c79c25fd16e/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.065071573Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/4a331860-fd72-42e5-801b-f6e26734b6a6/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.065458162Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.06582716Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"d265dd7c-f0fc-44fb-9f04-f32414cf0686","remote_addr":"[fd00:1122:3344:101::1]:38002","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":9192873,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.067905227Z","hostname":"buskin","pid":1566,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:658"} {"msg":"failed to write DNS configuration (will retry in 146.891808ms)","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:53.394162389Z","hostname":"buskin","pid":1566,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:678","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): client error (Connect): tcp connect error: Connection refused (os error 146): Connection refused (os error 146)"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.560255893Z","hostname":"buskin","pid":1566,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:658"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.608222945Z","hostname":"buskin","pid":1566,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:658"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.656468314Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:694"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.656481554Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.688585166Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.689537767Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.689545188Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 4ffccf6c-3438-4d74-b934-174aa54ef45c (service), filesystem_pool: Some(ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 }, image_source: InstallDataset }, OmicronZoneConfig { id: 1cb0bf88-9c23-4d9e-90d4-741665f48d09 (service), filesystem_pool: Some(ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 }, image_source: InstallDataset }, OmicronZoneConfig { id: 92ca8da9-bfa1-4f67-902e-4f82e77821d6 (service), filesystem_pool: Some(ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 }, image_source: InstallDataset }, OmicronZoneConfig { id: c755e1da-896c-4f49-9443-a651a42daa84 (service), filesystem_pool: Some(ZpoolName { id: 7dd4c27f-a9f0-4c2e-a0e0-1ad507079428 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2ef0dd69-74ea-4a66-998e-ec0e699d9744, kind: Service { id: c755e1da-896c-4f49-9443-a651a42daa84 }, name: Name(\"ntp-c755e1da-896c-4f49-9443-a651a42daa84\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 155, 29])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } }, image_source: InstallDataset }] }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.718693292Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.718829185Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:44255"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.719157582Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.719166413Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: a595bf70-cbb4-4ef8-8714-4d4d03d8ec49 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.730144932Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.730183413Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 1f1780df-c4f8-4ea2-9680-811efa3a608f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.738313469Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.738345719Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: f628b386-82f1-43d6-bae7-1b54d721271c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.745750558Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/651ad8e9-444b-45dc-9dfc-4c79c25fd16e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.745792529Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/4a331860-fd72-42e5-801b-f6e26734b6a6/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.745936822Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"ensure_all_omicron_zones: request reconciliation done","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.746031485Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4918","num_zones_updated":0,"num_zones_to_be_added":1,"num_zones_to_be_removed":0} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.746045485Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.75725385Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.789525975Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.790273492Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.790288102Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.819909557Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:461","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 155, 29]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.819917988Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1320","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.820235585Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.820245275Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.820501001Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.822178699Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.84725115Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:53.864286478Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:54.527902464Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84 zone","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:54.527990566Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:489"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84\") failed: Property not found. retry in 73.830218ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:54.657259881Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84\") failed: Property not found. retry in 145.758268ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:54.742984874Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84\") failed: Property not found. retry in 122.450685ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:54.899933649Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84\") failed: Property not found. retry in 328.037856ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:55.03212258Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84\") failed: Property not found. retry in 873.510505ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:55.370545859Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:54:56.137604842Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13828796589641439241"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:54:56.137748295Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17564117138543498946"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84\") failed: Property not found. retry in 1.249715291s","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:56.256318386Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:54:56.326562886Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:54:56.326693119Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84\") failed: Property not found. retry in 1.293331578s","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:57.515134861Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service svc:/milestone/single-user:default in zone Some(\"oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84\") failed: Property not found. retry in 729.664251ms","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:58.817881796Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:59.567723507Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3377","zone":"oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84"} {"msg":"Writing ledger to /pool/int/651ad8e9-444b-45dc-9dfc-4c79c25fd16e/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:59.567791438Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/4a331860-fd72-42e5-801b-f6e26734b6a6/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:59.568121676Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:59.568459494Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"b0834ba3-9efc-43fe-9482-b888cbee42fe","remote_addr":"[fd00:1122:3344:101::1]:44255","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":5849523,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:59.570636563Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:735"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:59.600839671Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:59.600990784Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:37478"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:54:59.611734019Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3971"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:59.611841552Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"8ca0a68b-bda8-410a-8d87-c33aa017a4f5","remote_addr":"[fd00:1122:3344:101::1]:37478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:855","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":10623,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:54:59.613987811Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:59.794056562Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:54:59.794284138Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:65297"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:02.848036869Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11428655806815006694"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:02.852290576Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"8878101146279988279"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:03.067644101Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17000508790196449848"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:03.803973094Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3971"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:03.804086587Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"049e89b8-d5c5-46d1-bd95-60e704abbcd3","remote_addr":"[fd00:1122:3344:101::1]:65297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:855","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009738,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:03.806537262Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:04.016014404Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:04.016194978Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:64089"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:04.025560752Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3971"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:04.025661154Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"1bcfd4b3-b52b-4f52-a9c5-f4ff8089b41d","remote_addr":"[fd00:1122:3344:101::1]:64089","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:855","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9401,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:04.027485406Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:04.150498828Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:04.150632101Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:61359"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_c755e1da-896c-4f49-9443-a651a42daa84': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:08.160070142Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3971"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:08.160198794Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"169abef2-7cae-4770-916e-e9ab41eb355f","remote_addr":"[fd00:1122:3344:101::1]:61359","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:855","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009455,"response_code":"500"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:08.162109338Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:08.162272822Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:08.162402235Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13828796589641439241"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:08.162527718Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17564117138543498946"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:08.163788596Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:08.376975552Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:08.377155867Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:37036"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:08.386647453Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"3a534484-5cc2-4117-8ea8-3c4bdb2a32af","remote_addr":"[fd00:1122:3344:101::1]:37036","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":9428,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:08.389841956Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:08.389862246Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:08.684822575Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:08.684951928Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:39552"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:08.693450721Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"a279e1dc-9feb-4f19-b745-9967e7f3442b","remote_addr":"[fd00:1122:3344:101::1]:39552","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":8372,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:08.695014237Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:08.695018217Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:08.865787327Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:08.865991792Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:44831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:08.87776158Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"7705cb53-7fff-4ca6-8a37-6b900b609aaa","remote_addr":"[fd00:1122:3344:101::1]:44831","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":11595,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:08.879326696Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:08.879333476Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:09.281567338Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:09.281712272Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:33489"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:09.290370079Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"de69a899-ab05-4cd5-9c95-1cc731cdefda","remote_addr":"[fd00:1122:3344:101::1]:33489","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":8579,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:09.291858152Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:09.291865443Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:09.57268768Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"18090473332170890861"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:09.572817953Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15635187575059277629"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:09.734758621Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:09.734896245Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:36110"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:09.74304197Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"769382e4-6cb7-4698-a69b-ff8c2ef10db2","remote_addr":"[fd00:1122:3344:101::1]:36110","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":8083,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:09.744204747Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:09.744210267Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:10.26590477Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:10.266094695Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:45264"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:10.27510984Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"ae8fa73a-7ea0-4ab7-a12f-570d3a907b5b","remote_addr":"[fd00:1122:3344:101::1]:45264","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":8918,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:10.276771218Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:10.276777068Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:11.015876994Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:11.016057628Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:54403"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:11.024980761Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"557493ba-3bc4-493a-b964-915784cf0eee","remote_addr":"[fd00:1122:3344:101::1]:54403","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":8849,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:11.028067772Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:11.028075492Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:11.826255994Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:11.826417778Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:43491"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:11.83531678Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"ae111452-eb59-48f4-ba7c-3c37393acaa1","remote_addr":"[fd00:1122:3344:101::1]:43491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":8829,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:11.836956887Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:11.836963388Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:12.841371147Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:12.841554882Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:36108"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:12.8498575Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11428655806815006694"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:12.850629108Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"fc756097-536f-4839-8287-a4ca16e33e0a","remote_addr":"[fd00:1122:3344:101::1]:36108","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":8994,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:12.852131662Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:12.852137632Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:12.854011485Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"8878101146279988279"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:13.068398658Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17000508790196449848"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:13.762228103Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:13.7624931Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:37102"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:13.769856237Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"243.238633ms"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:13.772341474Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"5a7d3574-8148-4c79-90d3-382ccc6bd05f","remote_addr":"[fd00:1122:3344:101::1]:37102","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":9619,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:13.774407991Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:13.774413051Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:15.033068282Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:15.033214265Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:62094"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:15.04268279Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"7a19721e-7bfc-4ad1-9e6a-b17f43800727","remote_addr":"[fd00:1122:3344:101::1]:62094","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":9412,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:15.046404695Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:15.046411746Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:15.781829008Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:200::2]:45875"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:15.781988861Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"677e03c3-39f9-4368-81be-ce33913097be","remote_addr":"[fdb0:200::2]:45875","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":63,"response_code":"200"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:16.035043796Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:16.035171199Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:37226"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:16.044237395Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"afe950f8-a5bc-47d1-9ddf-aa66601c916c","remote_addr":"[fd00:1122:3344:101::1]:37226","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":9004,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:16.046945877Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:16.046954357Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:18.16482661Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:18.164940183Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:18.165063845Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13828796589641439241"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:18.165185298Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17564117138543498946"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:18.283303609Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:18.283498673Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:33199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:18.292818345Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"103c891a-4783-445d-a36a-b52168502e59","remote_addr":"[fd00:1122:3344:101::1]:33199","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":9245,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:18.294577936Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:18.294582866Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:19.574718806Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"18090473332170890861"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:19.574998992Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15635187575059277629"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:20.629056343Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:20.629247587Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:41277"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:20.638346354Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"3b76f993-8b69-449c-a1f0-e81e14f37a73","remote_addr":"[fd00:1122:3344:101::1]:41277","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":8996,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:20.640300659Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:20.640307329Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:22.819261064Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:22.819421657Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:52035"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:22.828600946Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"8e0af6ea-7a3c-4056-b650-4d3bc3358b4e","remote_addr":"[fd00:1122:3344:101::1]:52035","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":9118,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:22.830311375Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:22.830317765Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:22.851361475Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11428655806815006694"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:22.854611309Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"8878101146279988279"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:23.069976615Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17000508790196449848"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:25.779831682Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:25.780032837Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:37753"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:25.789014752Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"bf85b238-6b84-4c44-97e7-0e517dea2e5a","remote_addr":"[fd00:1122:3344:101::1]:37753","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":8901,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:25.790871054Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:25.790876574Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:28.16700165Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:28.167118432Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:28.167233485Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13828796589641439241"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:28.167348388Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17564117138543498946"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:29.04942783Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"311.672042ms"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:29.576485546Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"18090473332170890861"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:29.576601909Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15635187575059277629"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:30.415702103Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:30.415848686Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:53831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:30.425338822Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"c1ceee2e-4a95-4023-a6cf-93675f4c1a6c","remote_addr":"[fd00:1122:3344:101::1]:53831","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":9422,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:30.428831192Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:30.428839952Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:32.853839191Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11428655806815006694"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:32.85596743Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"8878101146279988279"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:33.071270524Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17000508790196449848"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:35.543500339Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:35.543703143Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:41032"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:35.552939564Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"e15bfd2d-1158-401c-9920-e769c0491b2a","remote_addr":"[fd00:1122:3344:101::1]:41032","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":9135,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:35.554696784Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:35.554701114Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:38.168500384Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:38.168939183Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:38.16921857Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"13828796589641439241"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:38.169894855Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"17564117138543498946"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:39.57840795Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"18090473332170890861"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:39.578531343Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15635187575059277629"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:41.556370206Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:41.55655599Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:64116"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:41.565980655Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"61c914b7-64cd-4d71-aa4e-0726d93ac897","remote_addr":"[fd00:1122:3344:101::1]:64116","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":9332,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:41.567818307Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:41.567822637Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:42.855948379Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11428655806815006694"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:42.857085575Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"8878101146279988279"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:43.07241878Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17000508790196449848"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:44.399154822Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"1.375854829s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:45.815482335Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:200::2]:47739"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:45.815561126Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"912ab35e-3c79-451f-98c4-10f08191873a","remote_addr":"[fdb0:200::2]:47739","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":33,"response_code":"200"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:47.21482312Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:47.214940742Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:60364"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:47.224405418Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"4590c114-ab83-4d48-a583-f5bedcacb2e9","remote_addr":"[fd00:1122:3344:101::1]:60364","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":9337,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:47.226811803Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:47.226818763Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:48.170506419Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:48.170630182Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:49.580227532Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"18090473332170890861"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:49.580340214Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15635187575059277629"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:52.857978885Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"11428655806815006694"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:52.858394375Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"8878101146279988279"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:53.073783101Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"17000508790196449848"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:54.673587303Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:54.673789608Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:60304"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:54.683155631Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"e06dd13e-9261-44ab-84ad-3750a51ae81b","remote_addr":"[fd00:1122:3344:101::1]:60304","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":9296,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:55:54.684910761Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:55:54.684915671Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:58.173443266Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:58.173555549Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:59.583020795Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"18090473332170890861"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:55:59.583539977Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"15635187575059277629"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:56:00.811494978Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"2.416486304s"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:56:00.989234197Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:56:00.989347589Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:63312"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:56:00.998320784Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"00b9df63-8406-4df4-854a-2962bc80dc47","remote_addr":"[fd00:1122:3344:101::1]:63312","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":8754,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:56:01.000659847Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:56:01.000666547Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:56:07.940311056Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:56:07.94048722Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:48635"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:56:07.949966386Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"6620ae53-b20c-42db-92ae-7a53a9711d4c","remote_addr":"[fd00:1122:3344:101::1]:48635","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":9407,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:56:07.952225247Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:56:07.952240148Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:56:08.175960113Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:56:08.176090927Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:56:15.848063016Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:200::2]:62031"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:56:15.848178029Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"0745e9a9-7fee-4a25-86d7-8435cda39639","remote_addr":"[fdb0:200::2]:62031","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":36,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:56:18.176884615Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:56:18.178022451Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:56:18.263569669Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"4.57409816s"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:56:24.997087992Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:56:24.997243376Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:33223"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:56:25.006469026Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"5c680084-fb97-4dda-82a2-03f012cc29e8","remote_addr":"[fd00:1122:3344:101::1]:33223","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":9173,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:56:25.008888661Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:56:25.008895891Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:56:28.178988602Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:56:28.179111745Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:56:37.873357811Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"11.653502151s"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:56:38.180500387Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:56:38.18061886Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:56:41.077273773Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:56:41.077477297Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:32797"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:56:41.087143737Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"f9b7b1f8-b851-424f-9255-024861b4e21a","remote_addr":"[fd00:1122:3344:101::1]:32797","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":9573,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:56:41.08945864Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:56:41.08947099Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:56:45.880891314Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:200::2]:60177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:56:45.880968606Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"12d1222f-860b-4a17-824d-14726132ea75","remote_addr":"[fdb0:200::2]:60177","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":30,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:56:48.182982273Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:56:48.183099856Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:56:50.40901602Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:56:50.409167824Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:53352"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:56:50.418040286Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"7efdaaeb-7f4d-4efe-ac09-3c380edc847b","remote_addr":"[fd00:1122:3344:101::1]:53352","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":8809,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:56:50.420566634Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:56:50.420574804Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:56:58.184825636Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:56:58.185106672Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:57:04.562747028Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"19.940907001s"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:57:05.571908706Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:57:05.572100451Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:52208"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:57:05.581802932Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"657f7123-ac95-4b5a-8a09-b139844a90ea","remote_addr":"[fd00:1122:3344:101::1]:52208","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":9606,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:57:05.584161325Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:57:05.584166815Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:57:08.186952914Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:57:08.187070197Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:57:15.914715085Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:200::2]:55586"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:57:15.914829787Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"c5cb9af4-b6dd-4e32-9a44-aa0ac86a6db3","remote_addr":"[fdb0:200::2]:55586","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":43,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:57:18.188633022Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:57:18.188753995Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:57:19.512951239Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:57:19.513064112Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:44686"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:57:19.522174859Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"2f480f84-13b1-4f46-9707-bfe64afe6f9a","remote_addr":"[fd00:1122:3344:101::1]:44686","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":8903,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:57:19.524658145Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:57:19.524665326Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:57:28.190540656Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:57:28.190645768Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:57:38.192376948Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:57:38.19248394Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:57:39.539051514Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"38.361977186s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:57:45.948620637Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:200::2]:45707"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:57:45.948699449Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"5e53497e-45ec-4474-af6a-a9d0507487a7","remote_addr":"[fdb0:200::2]:45707","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":32,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:57:48.194942986Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:57:48.195066129Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:57:49.330323429Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:57:49.330437071Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:49804"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:57:49.340221244Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"ff65d402-6bc2-405e-8811-142ec1ae80a7","remote_addr":"[fd00:1122:3344:101::1]:49804","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":9639,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:57:49.344274197Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:57:49.344285477Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:57:58.197189607Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:57:58.197344541Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:58:08.199719125Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:58:08.199841198Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:58:15.983959052Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:200::2]:36252"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:58:15.984047544Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"59976781-f095-415d-b254-2a9accecce95","remote_addr":"[fdb0:200::2]:36252","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":33,"response_code":"200"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:58:16.265424323Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:58:16.265623138Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:35917"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:58:16.275007752Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"6bb7602e-83fe-4501-9ccb-747f6aa3cd21","remote_addr":"[fd00:1122:3344:101::1]:35917","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":9282,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:58:16.277611651Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:58:16.277616831Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:58:18.202180771Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:58:18.202297704Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:58:28.204403461Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:58:28.204511064Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:58:32.935908631Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"78.715950013s"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:58:38.206585351Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:58:38.206699204Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:58:46.01768772Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:200::2]:55464"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:58:46.017766982Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"cedd6478-1226-420b-ae2d-ca863a9a3aea","remote_addr":"[fdb0:200::2]:55464","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":30,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:58:48.208651698Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:58:48.209323553Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:58:58.210707595Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:58:58.210856298Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:58:58.797538033Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:58:58.797653455Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:54252"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:58:58.806407135Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"e72fa61d-8e97-41ae-9f43-e7369e52a262","remote_addr":"[fd00:1122:3344:101::1]:54252","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":8610,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:58:58.808654196Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:58:58.808658496Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:59:08.212324552Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:59:08.212430084Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:59:16.052588885Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:200::2]:39838"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:59:16.052721618Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"a98fc711-7544-4043-b492-a4d56caa17a6","remote_addr":"[fdb0:200::2]:39838","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":44,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:59:18.214630235Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:59:18.21487355Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:59:28.216825134Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:59:28.21708631Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:59:34.320340986Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:59:34.320455689Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:61363"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:59:34.329022774Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"7397700d-569d-46a8-b489-4aabc9065e6c","remote_addr":"[fd00:1122:3344:101::1]:61363","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":8432,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:59:34.331330577Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T00:59:34.331337857Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:59:38.218625295Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:59:38.218758778Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:59:39.773584996Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:59:46.085982706Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:200::2]:32794"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T00:59:46.086113699Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"5fe7f045-5127-4413-87bb-0d3c2abd0395","remote_addr":"[fdb0:200::2]:32794","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":46,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:59:48.221087412Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:59:48.221208964Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:59:58.224187432Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T00:59:58.224295764Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T01:00:06.685444291Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"84.14154749s"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:00:08.226776081Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:00:08.226891494Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T01:00:16.120887501Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:200::2]:35598"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T01:00:16.120964583Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"de23da72-798c-46a7-8254-6fc693beedcf","remote_addr":"[fdb0:200::2]:35598","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":27,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:00:18.228207114Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:00:18.228425249Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:00:28.230873414Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:00:28.230982627Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:00:38.232805218Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:00:38.232945601Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T01:00:46.155355266Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:200::2]:35972"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T01:00:46.155441808Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"d77baaae-003f-4daf-8c7d-ca6a7b152d74","remote_addr":"[fdb0:200::2]:35972","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":28,"response_code":"200"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T01:00:46.552334449Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T01:00:46.552550194Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:63761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T01:00:46.561688622Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"21048282-9a08-4e14-a669-fb7089510c7b","remote_addr":"[fd00:1122:3344:101::1]:63761","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":9047,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T01:00:46.564228Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T01:00:46.56423427Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:00:48.234985548Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:00:48.235110311Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:00:58.236552824Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:00:58.236823Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:01:08.238390575Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:01:08.238493418Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T01:01:16.189856072Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:200::2]:43710"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T01:01:16.189991275Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"c96042de-7495-4fb2-8d51-58c7f8279ef9","remote_addr":"[fdb0:200::2]:43710","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":47,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:01:18.24078631Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:01:18.240899243Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:01:28.242336866Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:01:28.242443228Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:01:38.24470925Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:01:38.244813592Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T01:01:45.862227839Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"262.361004088s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T01:01:46.224246086Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:200::2]:56215"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T01:01:46.224334558Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"1b0b8131-b7d4-41b5-97e3-b7fc4a082b7a","remote_addr":"[fdb0:200::2]:56215","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":34,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:01:48.246679604Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:01:48.246786657Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T01:01:54.223153722Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T01:01:54.223286965Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fd00:1122:3344:101::1]:41280"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T01:01:54.234588523Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"35261378-6961-4c91-8499-cf7e34cec6f5","remote_addr":"[fd00:1122:3344:101::1]:41280","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":11085,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T01:01:54.237019748Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-02-22T01:01:54.237029409Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:01:58.24908606Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:01:58.249216042Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:02:08.251838952Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:02:08.251977835Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T01:02:16.256744706Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:200::2]:55703"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T01:02:16.256858488Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"5144fcd3-9b6f-4c95-bc88-b5e7d838d945","remote_addr":"[fdb0:200::2]:55703","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":44,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:02:18.253586702Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:02:18.253735645Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:02:28.255794852Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:02:28.256058308Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:02:38.257764337Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:02:38.257975252Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T01:02:46.289363109Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:200::2]:43566"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T01:02:46.289432331Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"4546499c-02a7-4ada-8720-bf8f9fa652ae","remote_addr":"[fdb0:200::2]:43566","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":27,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:02:48.259085237Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:02:48.25922742Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:02:58.261646235Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:02:58.261799789Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:03:08.26405157Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:03:08.264297276Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T01:03:16.321079351Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:200::2]:63825"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T01:03:16.321167983Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"edf0a0bf-af4a-4861-ae55-dffd2d513fcc","remote_addr":"[fdb0:200::2]:63825","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":28,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:03:18.265297778Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:03:18.265509273Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:03:28.266860474Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:03:28.266965017Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:03:38.26841902Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:03:38.268533762Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T01:03:46.353858208Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:1025","remote_addr":"[fdb0:200::2]:45434"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-02-22T01:03:46.353988321Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"a3bb9414-85b1-4efe-8fc2-f31461f93dc6","remote_addr":"[fdb0:200::2]:45434","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.15.1/src/server.rs:867","latency_us":53,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:03:48.270596089Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"534409360623082643"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-02-22T01:03:48.270711612Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"46437054-bf3d-4063-a6ee-f11c748dcba3","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5781159090361636012"}