[ Jan 11 01:17:32 Enabled. ] [ Jan 11 01:17:32 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Jan 11 01:17:32 Method "start" exited with status 0. ] [ Jan 11 01:17:32 Rereading configuration. ] [ Jan 11 01:17:32 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:32.686820643Z","hostname":"sock","pid":1566,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:32.686998145Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:32.738272975Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:32.738311435Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:32.853923089Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:32.862532301Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:32.870353874Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:33.470808878Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:200::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:33.613916288Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:126"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:33.613965399Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:138"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:33.61407021Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:151"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:33.614157121Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:33.614167871Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:33.614190391Z","hostname":"sock","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:717"} {"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:33.617296868Z","hostname":"sock","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:624"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:33.617309318Z","hostname":"sock","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:766"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:33.617329108Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:187"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:33.617336608Z","hostname":"sock","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:825"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:33.617352069Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:33.617398909Z","hostname":"sock","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:33.617567811Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:33.617609852Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:33.617629042Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:33.629807737Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:33.629840007Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"2026aef5-ecc1-428c-aac9-f90b84f99582 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:33.730461663Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 2026aef5-ecc1-428c-aac9-f90b84f99582 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.105374596Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.105410396Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.105415737Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.105429186Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.105439217Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.105444847Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.105455987Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.114271432Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.114296102Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"9404c118-e76c-41a3-b09c-850793e381f3 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.201914623Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 9404c118-e76c-41a3-b09c-850793e381f3 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567458646Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567508856Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 2026aef5-ecc1-428c-aac9-f90b84f99582 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 9404c118-e76c-41a3-b09c-850793e381f3 (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567539827Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567547287Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567556987Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567564247Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567572087Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567643088Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 2026aef5-ecc1-428c-aac9-f90b84f99582 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 9404c118-e76c-41a3-b09c-850793e381f3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567661408Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567666818Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567670818Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567684229Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567714889Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567719609Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 2026aef5-ecc1-428c-aac9-f90b84f99582 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 9404c118-e76c-41a3-b09c-850793e381f3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567736339Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567741909Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567746049Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567751199Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567755579Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.56776908Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 2026aef5-ecc1-428c-aac9-f90b84f99582 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 9404c118-e76c-41a3-b09c-850793e381f3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.56778614Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.56779737Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.56780159Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.56780519Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.56780938Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.56781262Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 2026aef5-ecc1-428c-aac9-f90b84f99582 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 9404c118-e76c-41a3-b09c-850793e381f3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.56782952Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.56783514Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567838991Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.56784316Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.5678478Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.56785146Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 2026aef5-ecc1-428c-aac9-f90b84f99582 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 9404c118-e76c-41a3-b09c-850793e381f3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567876481Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567881311Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567885351Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567891591Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567895781Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567910381Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567915521Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 2026aef5-ecc1-428c-aac9-f90b84f99582 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 9404c118-e76c-41a3-b09c-850793e381f3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567945922Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567951672Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567956272Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567959652Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567963232Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 2026aef5-ecc1-428c-aac9-f90b84f99582 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 9404c118-e76c-41a3-b09c-850793e381f3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.567995452Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.568013322Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.568018553Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.568022933Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.568026583Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.568031073Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.568034453Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 2026aef5-ecc1-428c-aac9-f90b84f99582 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 9404c118-e76c-41a3-b09c-850793e381f3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.568065373Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.568070133Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.568074553Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.568083343Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:97"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.568094223Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:99"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.568097983Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 2026aef5-ecc1-428c-aac9-f90b84f99582 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 9404c118-e76c-41a3-b09c-850793e381f3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.568126024Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.568130734Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.568134014Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.568137404Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:211"} {"msg":"No ledger in /pool/int/2026aef5-ecc1-428c-aac9-f90b84f99582/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.568152274Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/9404c118-e76c-41a3-b09c-850793e381f3/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.568180184Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.568185884Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/2026aef5-ecc1-428c-aac9-f90b84f99582/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.568194855Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/9404c118-e76c-41a3-b09c-850793e381f3/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.568200305Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.568207235Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.568238245Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:216"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.568242675Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:231"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.578964733Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:748"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.578996833Z","hostname":"sock","pid":1566,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999920239s","next_cleanup":"Instant { tv_sec: 1127, tv_nsec: 353842733 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.579034724Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3973"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.579087004Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/2026aef5-ecc1-428c-aac9-f90b84f99582/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.579141815Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/9404c118-e76c-41a3-b09c-850793e381f3/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.581923788Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.582300512Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.583962362Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:51","local_addr":"[fdb0:200::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: Some(\n None,\n ),\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n tx_eq: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.584204485Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1204"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.584210505Z","hostname":"sock","pid":1566,"component":"DnsResolver","component":"RSS","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.584336027Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.584375327Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/2026aef5-ecc1-428c-aac9-f90b84f99582/config/rss-started.marker","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.584426997Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/9404c118-e76c-41a3-b09c-850793e381f3/config/rss-started.marker","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.584457768Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/2026aef5-ecc1-428c-aac9-f90b84f99582/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.584471708Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/9404c118-e76c-41a3-b09c-850793e381f3/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.584483018Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS not previously run. Creating plans.","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.58464527Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1253"} {"msg":"Writing ledger to /pool/int/2026aef5-ecc1-428c-aac9-f90b84f99582/config/.rss-started.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.58465623Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/9404c118-e76c-41a3-b09c-850793e381f3/config/.rss-started.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.584883953Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.585013364Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1285"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.585018315Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:66"} {"msg":"Creating plan for the sled at fdb0:200::1","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.585023605Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:39"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.585064405Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1316"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.585118116Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/2026aef5-ecc1-428c-aac9-f90b84f99582/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.585131456Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/9404c118-e76c-41a3-b09c-850793e381f3/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.585295678Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.585429109Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.585523181Z","hostname":"sock","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:200::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: dfd15883-50bd-419a-82e1-b0ed2f66c5e6 (sled), rack_id: 90cda70f-4d85-4179-bfe4-b1e37ebf7366, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.585907015Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4270"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.585974216Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4664"} {"msg":"Loaded keys and certs","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.590613401Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:200::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/client.rs:50"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.596181227Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:200::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.596291209Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:200::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.600392247Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.600465718Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.600536029Z","hostname":"sock","pid":1566,"remote_addr":"[fdb0:200::1]:42451","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:74"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: dfd15883-50bd-419a-82e1-b0ed2f66c5e6 (sled), rack_id: 90cda70f-4d85-4179-bfe4-b1e37ebf7366, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.600694271Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:362"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.600709881Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:377"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.600760791Z","hostname":"sock","pid":1566,"file":"sled-agent/src/server.rs:44"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.600766052Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.600770592Z","hostname":"sock","pid":1566,"component":"DnsResolver","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.600779272Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:813"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.600830163Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"No ledger in /pool/int/2026aef5-ecc1-428c-aac9-f90b84f99582/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.600863283Z","hostname":"sock","pid":1566,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/9404c118-e76c-41a3-b09c-850793e381f3/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.600897853Z","hostname":"sock","pid":1566,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.600912773Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:802"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.600917584Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:830"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.600924713Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:596"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.603641546Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.603688426Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:701","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.615984652Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.636278693Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.653124334Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.684186703Z","hostname":"sock","pid":1566,"sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:412"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.684231663Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.684236983Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.684269624Z","hostname":"sock","pid":1566,"sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:424"} {"msg":"Failed enable ddm stats (will retry after 137.770748ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.687101207Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d196ae2b-7831-4e65-802a-f2868e8a9233\", \"content-length\": \"124\", \"date\": \"Sat, 11 Jan 2025 01:17:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d196ae2b-7831-4e65-802a-f2868e8a9233\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_2026aef5-ecc1-428c-aac9-f90b84f99582/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.697799464Z","hostname":"sock","pid":1566,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_2026aef5-ecc1-428c-aac9-f90b84f99582/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.721500466Z","hostname":"sock","pid":1566,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_2026aef5-ecc1-428c-aac9-f90b84f99582/swap\"","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.721550837Z","hostname":"sock","pid":1566,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.726638297Z","hostname":"sock","pid":1566,"sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:439"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.726646307Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:129"} {"msg":"Ensuring dataset oxi_2026aef5-ecc1-428c-aac9-f90b84f99582/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.726654197Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:139"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.78996611Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:174"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.826118689Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 116.41878ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.826739487Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"53929db6-45fd-4e39-9346-1d49fd91d675\", \"content-length\": \"124\", \"date\": \"Sat, 11 Jan 2025 01:17:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"53929db6-45fd-4e39-9346-1d49fd91d675\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.943611935Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 88.664611ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:34.94403851Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d0d77627-397d-4a0c-b891-9ef37a6c54ec\", \"content-length\": \"124\", \"date\": \"Sat, 11 Jan 2025 01:17:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d0d77627-397d-4a0c-b891-9ef37a6c54ec\" }"} {"msg":"Mounting oxi_2026aef5-ecc1-428c-aac9-f90b84f99582/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:35.009506707Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:182"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:35.021795833Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:190"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:35.021884114Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:190"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:35.021910654Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:190"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:35.021934205Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:196"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:35.033928257Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 208.32102ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:35.034417823Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"44a03688-0f2b-4737-affb-379ca984aa65\", \"content-length\": \"124\", \"date\": \"Sat, 11 Jan 2025 01:17:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"44a03688-0f2b-4737-affb-379ca984aa65\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:35.243880642Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 272.804831ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:35.24461444Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"07f3596b-101c-452c-898a-ee74a2e3dac7\", \"content-length\": \"124\", \"date\": \"Sat, 11 Jan 2025 01:17:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"07f3596b-101c-452c-898a-ee74a2e3dac7\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:35.518031809Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 272.74824ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:35.518777948Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5f7a6678-b0d6-4982-a221-2a094d0f8e75\", \"content-length\": \"124\", \"date\": \"Sat, 11 Jan 2025 01:17:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5f7a6678-b0d6-4982-a221-2a094d0f8e75\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:35.773695756Z","hostname":"sock","pid":1566,"sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:93"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:35.779257082Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:48724","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327","component":"producer-server","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:35.779305373Z","hostname":"sock","pid":1566,"component":"internal-dns-resolver","file":"internal-dns/resolver/src/resolver.rs:111","component":"producer-server","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:35.779364724Z","hostname":"sock","pid":1566,"file":"oximeter/producer/src/lib.rs:279","component":"producer-server","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:279","interval":"30s","address":"[fd00:1122:3344:101::1]:48724","producer_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:35.780336075Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"152.599092ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:35.781339487Z","hostname":"sock","pid":1566,"component":"VmmReservoirManager","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:35.781345857Z","hostname":"sock","pid":1566,"component":"VmmReservoirManager","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:35.781406218Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:932","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:35.781412298Z","hostname":"sock","pid":1566,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:35.793258659Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:35.934558177Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"488.560286ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:36.424295505Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"542.94595ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:36.969064826Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.060012825s"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:38.278882258Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2694"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:38.278905208Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2939"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:38.278910248Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2564"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:38.278920368Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2611"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:38.278963139Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3020"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:38.278968059Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:38.280863561Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2864"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:38.282146046Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2605"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:38.282237067Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:38.282397609Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 45.179923ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:38.616652771Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 94.243242ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:38.826041738Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 283.479922ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:38.997379373Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:39.030500407Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"4.296214923s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 350.572941ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:39.341035176Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.057520375s","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:39.819691873Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.3815016s","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:40.98947034Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.221295414s","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:42.565207251Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:43.328311887Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"11.013286772s"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/dfd15883-50bd-419a-82e1-b0ed2f66c5e6)","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:44.311174313Z","hostname":"sock","pid":1566,"component":"NexusNotifierTask","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:44.340928397Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:44.34124198Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12348","component":"dropshot (Repo Depot)","component":"ArtifactStore","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:44.342409464Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:44.342435635Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:432"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:44.342446355Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Writing ledger to /pool/int/2026aef5-ecc1-428c-aac9-f90b84f99582/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:44.342457345Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/9404c118-e76c-41a3-b09c-850793e381f3/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:44.342622957Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:44.342801799Z","hostname":"sock","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:44.3428548Z","hostname":"sock","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.2.100, vlan_id: None, rib_priority: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: Some(None), bgp_peers: [], autoneg: false, lldp: None, tx_eq: None }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:44.342864969Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3973"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:44.34289478Z","hostname":"sock","pid":1566,"remote_addr":"[fdb0:200::1]:42451","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:81"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:44.543309501Z","hostname":"sock","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:117","peer":"[fdb0:200::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:44.543390362Z","hostname":"sock","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:252","target_sled":"[fdb0:200::1]:12346"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:44.548470682Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4294","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/dfd15883-50bd-419a-82e1-b0ed2f66c5e6)","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:17:45.517039019Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:45.519130584Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:42613"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:45.519420517Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:45.519428217Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/2026aef5-ecc1-428c-aac9-f90b84f99582/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:45.519513888Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/9404c118-e76c-41a3-b09c-850793e381f3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:45.519547799Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:45.51968159Z","hostname":"sock","pid":1566,"uri":"/inventory","method":"GET","req_id":"4e36db09-2ffd-4438-b6eb-5b27fa234dfb","remote_addr":"[fd00:1122:3344:101::1]:42613","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":375,"response_code":"200"} {"msg":"refreshed zone-network-setup service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:45.751426144Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4343"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:45.751487054Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4401"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:17:45.983427469Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:17:45.983543991Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.213962508Z","hostname":"sock","pid":1566,"component":"VmmReservoirManager","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.228045756Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:34307"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.228258188Z","hostname":"sock","pid":1566,"uri":"/sled-role","method":"GET","req_id":"6627f099-e2f4-42d2-b514-cf924e97a96a","remote_addr":"[fd00:1122:3344:101::1]:34307","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":77,"response_code":"200"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(2), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 69dbdd0a-897c-4e99-a470-a91142cf83fa (physical_disk), pool_id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: e2fde8f7-1205-4d60-ba0f-04707302a821 (physical_disk), pool_id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: a5a99cb6-3ab1-4115-b501-c7c97275d683 (physical_disk), pool_id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: e143a2be-1010-4476-9c0d-0e2241e91921 (physical_disk), pool_id: 1d1a100e-9784-4f32-981f-1a3200e989b4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 27eb8077-ecfb-4b20-b5be-4d91c4c68586 (physical_disk), pool_id: f5232c69-2469-4824-a4f8-901936b93f3d (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: c8d3e277-d7d9-4b9c-a109-67853d99b0fe (physical_disk), pool_id: 70afe259-ef7a-419a-97ed-388d46a1cb17 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 1f22332d-7a1e-4714-b86c-21b6381791f9 (physical_disk), pool_id: 0a1d8462-3db8-4b03-acdf-9503499e711d (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 3a48a252-fa8d-4d14-a788-4b4abb5b2cd4 (physical_disk), pool_id: 094891fc-36ea-4f74-b7ba-91566beb4534 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: e3ef064c-7782-4aab-9698-006236d975a9 (physical_disk), pool_id: a864efea-d3d9-4be0-84a5-f7e73206c783 (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.259783863Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:385"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.259881204Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:43734"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.260172137Z","hostname":"sock","pid":1566,"sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:893"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(2), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 69dbdd0a-897c-4e99-a470-a91142cf83fa (physical_disk), pool_id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: e2fde8f7-1205-4d60-ba0f-04707302a821 (physical_disk), pool_id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: a5a99cb6-3ab1-4115-b501-c7c97275d683 (physical_disk), pool_id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: e143a2be-1010-4476-9c0d-0e2241e91921 (physical_disk), pool_id: 1d1a100e-9784-4f32-981f-1a3200e989b4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 27eb8077-ecfb-4b20-b5be-4d91c4c68586 (physical_disk), pool_id: f5232c69-2469-4824-a4f8-901936b93f3d (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: c8d3e277-d7d9-4b9c-a109-67853d99b0fe (physical_disk), pool_id: 70afe259-ef7a-419a-97ed-388d46a1cb17 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 1f22332d-7a1e-4714-b86c-21b6381791f9 (physical_disk), pool_id: 0a1d8462-3db8-4b03-acdf-9503499e711d (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 3a48a252-fa8d-4d14-a788-4b4abb5b2cd4 (physical_disk), pool_id: 094891fc-36ea-4f74-b7ba-91566beb4534 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: e3ef064c-7782-4aab-9698-006236d975a9 (physical_disk), pool_id: a864efea-d3d9-4be0-84a5-f7e73206c783 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.260246508Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/2026aef5-ecc1-428c-aac9-f90b84f99582/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.260257538Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/9404c118-e76c-41a3-b09c-850793e381f3/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.260268439Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.260276348Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1298"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.260306469Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.260313739Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.260335299Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.260342789Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.270735443Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.270760333Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.365403088Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.374101161Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.374175882Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.374186802Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.374196632Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.374209012Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.374326174Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.374336934Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.40856107Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.417110602Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/dfd15883-50bd-419a-82e1-b0ed2f66c5e6)","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:17:46.519035793Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Looking for unencrypted datasets in oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.552870795Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.560531636Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.560579396Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.560608177Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.560616817Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.560623827Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.570457534Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.570480874Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"8d41776e-a84a-44c7-a848-bc443cc63109 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.665890498Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.674429229Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.67448006Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.67449023Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.67452378Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.67453747Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.674676192Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.674684762Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.709707018Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.717709803Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_8d41776e-a84a-44c7-a848-bc443cc63109","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.855634982Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.863775639Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.863817229Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.863825919Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.863836149Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.86385273Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.873928779Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.87395692Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.968189899Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.976257515Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.976295395Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.976305955Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.976349386Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.976407997Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.976532658Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:46.976541058Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.011798747Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.020610592Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.162507628Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.170650684Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.170689515Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.170697695Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.170703755Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.170710695Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.180520932Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.180543852Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"1d1a100e-9784-4f32-981f-1a3200e989b4 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.274945184Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 1d1a100e-9784-4f32-981f-1a3200e989b4 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.28307931Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.283119731Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.283128911Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.283181592Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.283193882Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.283357823Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.283366834Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.317980385Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_1d1a100e-9784-4f32-981f-1a3200e989b4/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.326405495Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_1d1a100e-9784-4f32-981f-1a3200e989b4","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.466871054Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.475119582Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.475164032Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.475174473Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.475183043Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.475230783Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.484860578Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.484880198Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"f5232c69-2469-4824-a4f8-901936b93f3d (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.58012384Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: f5232c69-2469-4824-a4f8-901936b93f3d (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.588694071Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.588730822Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.588741712Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.588776252Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.588795332Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.588913184Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.588921474Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/dfd15883-50bd-419a-82e1-b0ed2f66c5e6)","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:17:47.592642088Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.624081172Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_f5232c69-2469-4824-a4f8-901936b93f3d/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.632585183Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_f5232c69-2469-4824-a4f8-901936b93f3d","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.770548702Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.77880542Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.778866221Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.778888881Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.778898901Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.778907311Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.788458525Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.788492865Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"70afe259-ef7a-419a-97ed-388d46a1cb17 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.880417347Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: 70afe259-ef7a-419a-97ed-388d46a1cb17 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.888150079Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.88818824Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.888194609Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.8882369Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.88824873Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.888421492Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.888436643Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.924464011Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_70afe259-ef7a-419a-97ed-388d46a1cb17/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:47.932701008Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_70afe259-ef7a-419a-97ed-388d46a1cb17","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.068848495Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.076598537Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.076636678Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.076645538Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.076652418Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.076691588Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.086359703Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.086396664Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"0a1d8462-3db8-4b03-acdf-9503499e711d (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.185539521Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 0a1d8462-3db8-4b03-acdf-9503499e711d (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.194590819Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.19463811Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.19465549Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.19467953Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.19469225Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.194839522Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.194848532Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.230282343Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_0a1d8462-3db8-4b03-acdf-9503499e711d/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.238261428Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_0a1d8462-3db8-4b03-acdf-9503499e711d","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.376843125Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.385654579Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.385681799Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.385687599Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.38569279Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.38569775Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.39584009Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.39585923Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"094891fc-36ea-4f74-b7ba-91566beb4534 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.490738108Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 094891fc-36ea-4f74-b7ba-91566beb4534 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.498633252Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.498654472Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.498659412Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.498674912Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.498716242Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.498851834Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.498859244Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.535924675Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_094891fc-36ea-4f74-b7ba-91566beb4534/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.543925739Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/dfd15883-50bd-419a-82e1-b0ed2f66c5e6)","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:17:48.594675742Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Looking for unencrypted datasets in oxp_094891fc-36ea-4f74-b7ba-91566beb4534","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.68954899Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.697891229Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.697940799Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.697950709Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.69795806Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.69796682Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.707993819Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.708030979Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"a864efea-d3d9-4be0-84a5-f7e73206c783 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.804752968Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: a864efea-d3d9-4be0-84a5-f7e73206c783 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.811804612Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.811837293Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.811846523Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.811898263Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.811933584Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.812049045Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.812058295Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.844345109Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_a864efea-d3d9-4be0-84a5-f7e73206c783/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.852761909Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_a864efea-d3d9-4be0-84a5-f7e73206c783","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:48.994767806Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.003178136Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.003208926Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/2026aef5-ecc1-428c-aac9-f90b84f99582/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.003289827Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.003410698Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(2), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 2026aef5-ecc1-428c-aac9-f90b84f99582 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 9404c118-e76c-41a3-b09c-850793e381f3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 1d1a100e-9784-4f32-981f-1a3200e989b4 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f5232c69-2469-4824-a4f8-901936b93f3d (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 70afe259-ef7a-419a-97ed-388d46a1cb17 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0a1d8462-3db8-4b03-acdf-9503499e711d (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 094891fc-36ea-4f74-b7ba-91566beb4534 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a864efea-d3d9-4be0-84a5-f7e73206c783 (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.003489519Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.003498039Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/9404c118-e76c-41a3-b09c-850793e381f3/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.003626021Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.003861454Z","hostname":"sock","pid":1566,"sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:897"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.003897574Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.003908595Z","hostname":"sock","pid":1566,"sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:923","generation":"Generation(2)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.003917604Z","hostname":"sock","pid":1566,"sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:928"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.003922865Z","hostname":"sock","pid":1566,"sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:933"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.003961525Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:122","gen":"Generation(2)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.003975595Z","hostname":"sock","pid":1566,"sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:939"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.004014506Z","hostname":"sock","pid":1566,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:799","gen":"Generation(2)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.004019686Z","hostname":"sock","pid":1566,"sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:944"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.004108247Z","hostname":"sock","pid":1566,"uri":"/omicron-physical-disks","method":"PUT","req_id":"41169f61-bf30-4488-8362-c6a25cbb0df6","remote_addr":"[fd00:1122:3344:101::1]:43734","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":2744126,"response_code":"200"} {"msg":"attempting to set up sled's datasets: DatasetsConfig { generation: Generation(1), datasets: {09b0c7ae-601d-42b3-8334-f0bae806ab74 (dataset): DatasetConfig { id: 09b0c7ae-601d-42b3-8334-f0bae806ab74 (dataset), name: DatasetName { pool_name: ZpoolName { id: f5232c69-2469-4824-a4f8-901936b93f3d (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_fa11dff5-c002-4f6a-8b1b-b726a1ca0a20\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0bfe0e66-2d27-46bc-8caf-06cd489a25b5 (dataset): DatasetConfig { id: 0bfe0e66-2d27-46bc-8caf-06cd489a25b5 (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0fa1cc1d-5155-47bd-b94a-71f9b1fb9b84 (dataset): DatasetConfig { id: 0fa1cc1d-5155-47bd-b94a-71f9b1fb9b84 (dataset), name: DatasetName { pool_name: ZpoolName { id: 094891fc-36ea-4f74-b7ba-91566beb4534 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 13e87a70-c620-488f-9c4c-52a3dd4c74b2 (dataset): DatasetConfig { id: 13e87a70-c620-488f-9c4c-52a3dd4c74b2 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 175c6b4c-9871-49c4-9f6d-262d07281ddd (dataset): DatasetConfig { id: 175c6b4c-9871-49c4-9f6d-262d07281ddd (dataset), name: DatasetName { pool_name: ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1c1e6f02-6343-40d6-9459-0ad7ebbe72dd (dataset): DatasetConfig { id: 1c1e6f02-6343-40d6-9459-0ad7ebbe72dd (dataset), name: DatasetName { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1c709747-81e4-4696-a312-d8fc3952bf42 (dataset): DatasetConfig { id: 1c709747-81e4-4696-a312-d8fc3952bf42 (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: TransientZone { name: \"oxz_oximeter_17bd3def-b7fb-4428-a2b5-5f1d745fd5a7\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 25ca6b38-5252-4b35-a88e-c539f69b81e0 (dataset): DatasetConfig { id: 25ca6b38-5252-4b35-a88e-c539f69b81e0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d1a100e-9784-4f32-981f-1a3200e989b4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_9eee15bb-10a1-49cf-8087-610615834db2\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 291f0653-83e9-4b2e-b2ca-f378f3ee968a (dataset): DatasetConfig { id: 291f0653-83e9-4b2e-b2ca-f378f3ee968a (dataset), name: DatasetName { pool_name: ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2d702f49-c939-49fa-a71c-59908195b58e (dataset): DatasetConfig { id: 2d702f49-c939-49fa-a71c-59908195b58e (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_dbcc85a3-d91b-402e-bbe0-02359124c83e\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 359fb39c-3752-4bd5-b0cf-d4273c8e9d01 (dataset): DatasetConfig { id: 359fb39c-3752-4bd5-b0cf-d4273c8e9d01 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 39a7645a-2aa7-401a-9d87-ca92258eea3b (dataset): DatasetConfig { id: 39a7645a-2aa7-401a-9d87-ca92258eea3b (dataset), name: DatasetName { pool_name: ZpoolName { id: 70afe259-ef7a-419a-97ed-388d46a1cb17 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_ae3bb366-dd85-4558-a521-f33b9626f0b1\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3a11fb65-1a6f-46b5-91b2-28f3add03a1a (dataset): DatasetConfig { id: 3a11fb65-1a6f-46b5-91b2-28f3add03a1a (dataset), name: DatasetName { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3f086a5d-8479-4acf-9d3a-f3e97c63a17e (dataset): DatasetConfig { id: 3f086a5d-8479-4acf-9d3a-f3e97c63a17e (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 46a77ce2-fe19-4301-a3df-76f05cf64ad7 (dataset): DatasetConfig { id: 46a77ce2-fe19-4301-a3df-76f05cf64ad7 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0a1d8462-3db8-4b03-acdf-9503499e711d (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4b96cd1d-d3c9-483f-b15d-b2c6178ea9d2 (dataset): DatasetConfig { id: 4b96cd1d-d3c9-483f-b15d-b2c6178ea9d2 (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_0278158b-ebb4-415c-bdce-d9833e816939\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4be48c53-1e9b-472d-b1dd-b0ea92506ee4 (dataset): DatasetConfig { id: 4be48c53-1e9b-472d-b1dd-b0ea92506ee4 (dataset), name: DatasetName { pool_name: ZpoolName { id: a864efea-d3d9-4be0-84a5-f7e73206c783 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 4c1c3fe9-b50d-4e3d-bb08-75de0ef7b512 (dataset): DatasetConfig { id: 4c1c3fe9-b50d-4e3d-bb08-75de0ef7b512 (dataset), name: DatasetName { pool_name: ZpoolName { id: 094891fc-36ea-4f74-b7ba-91566beb4534 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4c40936e-b09e-4620-a87b-09f3378f9031 (dataset): DatasetConfig { id: 4c40936e-b09e-4620-a87b-09f3378f9031 (dataset), name: DatasetName { pool_name: ZpoolName { id: a864efea-d3d9-4be0-84a5-f7e73206c783 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_68e8e65f-6191-43af-b0e4-85cba11d9294\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4fd18126-78b3-4507-b435-97f1efede5d8 (dataset): DatasetConfig { id: 4fd18126-78b3-4507-b435-97f1efede5d8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5344f04e-8cc1-483e-818b-3f158258d437 (dataset): DatasetConfig { id: 5344f04e-8cc1-483e-818b-3f158258d437 (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5c9f7970-3683-4463-9891-b44549db7a6e (dataset): DatasetConfig { id: 5c9f7970-3683-4463-9891-b44549db7a6e (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d1a100e-9784-4f32-981f-1a3200e989b4 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5cae68d6-bc26-4639-ad2d-229af4965de7 (dataset): DatasetConfig { id: 5cae68d6-bc26-4639-ad2d-229af4965de7 (dataset), name: DatasetName { pool_name: ZpoolName { id: a864efea-d3d9-4be0-84a5-f7e73206c783 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_ca441e07-5182-40ca-bf73-5e03b6c846d9\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 695ba87e-8f5d-4571-bb5b-e28635a25d1d (dataset): DatasetConfig { id: 695ba87e-8f5d-4571-bb5b-e28635a25d1d (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6b882b89-3173-44ac-826e-c727dee18043 (dataset): DatasetConfig { id: 6b882b89-3173-44ac-826e-c727dee18043 (dataset), name: DatasetName { pool_name: ZpoolName { id: f5232c69-2469-4824-a4f8-901936b93f3d (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 71175310-7d13-498e-96d4-c720233cf99a (dataset): DatasetConfig { id: 71175310-7d13-498e-96d4-c720233cf99a (dataset), name: DatasetName { pool_name: ZpoolName { id: 70afe259-ef7a-419a-97ed-388d46a1cb17 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 740c3ea5-134c-41a4-a0c2-f625e5be0130 (dataset): DatasetConfig { id: 740c3ea5-134c-41a4-a0c2-f625e5be0130 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 74439e02-29e6-43c9-8771-1221c3e33988 (dataset): DatasetConfig { id: 74439e02-29e6-43c9-8771-1221c3e33988 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 747bdce5-004e-4038-8931-6ce5eaa70188 (dataset): DatasetConfig { id: 747bdce5-004e-4038-8931-6ce5eaa70188 (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 76242899-0849-4deb-a960-ce3e00a06ba8 (dataset): DatasetConfig { id: 76242899-0849-4deb-a960-ce3e00a06ba8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7739caad-267b-467b-8fd5-a50a0096bd4a (dataset): DatasetConfig { id: 7739caad-267b-467b-8fd5-a50a0096bd4a (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d1a100e-9784-4f32-981f-1a3200e989b4 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7ab33682-ac74-48a4-8e88-a54322dbce8c (dataset): DatasetConfig { id: 7ab33682-ac74-48a4-8e88-a54322dbce8c (dataset), name: DatasetName { pool_name: ZpoolName { id: f5232c69-2469-4824-a4f8-901936b93f3d (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 808898e9-4271-442c-8793-5cc9c1376152 (dataset): DatasetConfig { id: 808898e9-4271-442c-8793-5cc9c1376152 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_95fbeccd-695b-4f6e-ae96-35054f4e2171\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 84345852-00ef-4077-a97e-21c8b3f9896f (dataset): DatasetConfig { id: 84345852-00ef-4077-a97e-21c8b3f9896f (dataset), name: DatasetName { pool_name: ZpoolName { id: f5232c69-2469-4824-a4f8-901936b93f3d (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_00711fcf-5ea5-4dbf-918c-e21236b2cac3\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 86147b90-245f-4d55-baee-ea0bb169d87f (dataset): DatasetConfig { id: 86147b90-245f-4d55-baee-ea0bb169d87f (dataset), name: DatasetName { pool_name: ZpoolName { id: f5232c69-2469-4824-a4f8-901936b93f3d (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 86a1286f-398c-4528-b313-5d2eb1b03de4 (dataset): DatasetConfig { id: 86a1286f-398c-4528-b313-5d2eb1b03de4 (dataset), name: DatasetName { pool_name: ZpoolName { id: 094891fc-36ea-4f74-b7ba-91566beb4534 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8701e2b8-78da-46c7-8957-25f526c725ec (dataset): DatasetConfig { id: 8701e2b8-78da-46c7-8957-25f526c725ec (dataset), name: DatasetName { pool_name: ZpoolName { id: a864efea-d3d9-4be0-84a5-f7e73206c783 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8bfdc6cb-8756-4ad3-a327-27f1df997f06 (dataset): DatasetConfig { id: 8bfdc6cb-8756-4ad3-a327-27f1df997f06 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_1877f8e9-a377-4960-91f4-09dd1c711f3b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8c405b05-f6c9-4658-b737-361075b5216f (dataset): DatasetConfig { id: 8c405b05-f6c9-4658-b737-361075b5216f (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d1a100e-9784-4f32-981f-1a3200e989b4 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8c69151f-038e-4a2b-8344-a8aae7b36024 (dataset): DatasetConfig { id: 8c69151f-038e-4a2b-8344-a8aae7b36024 (dataset), name: DatasetName { pool_name: ZpoolName { id: f5232c69-2469-4824-a4f8-901936b93f3d (zpool), kind: External }, kind: TransientZone { name: \"oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 935e6b13-1747-44c8-9f5a-1cb16f0f54a3 (dataset): DatasetConfig { id: 935e6b13-1747-44c8-9f5a-1cb16f0f54a3 (dataset), name: DatasetName { pool_name: ZpoolName { id: a864efea-d3d9-4be0-84a5-f7e73206c783 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9862a669-1af0-4cd7-a424-708337cd7a44 (dataset): DatasetConfig { id: 9862a669-1af0-4cd7-a424-708337cd7a44 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_dbbd09fb-c1a9-46dd-b9f4-782842395cbe\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 998e3b4e-5e5d-4e8c-ad7c-236c106bbd10 (dataset): DatasetConfig { id: 998e3b4e-5e5d-4e8c-ad7c-236c106bbd10 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d1a100e-9784-4f32-981f-1a3200e989b4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_38bf453c-46b6-4fd8-b901-1a35d301feef\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9f6329a2-cbdc-4c02-b4da-15bb972ccf1a (dataset): DatasetConfig { id: 9f6329a2-cbdc-4c02-b4da-15bb972ccf1a (dataset), name: DatasetName { pool_name: ZpoolName { id: 70afe259-ef7a-419a-97ed-388d46a1cb17 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b273b974-e613-4c89-a061-86dd9290f521 (dataset): DatasetConfig { id: b273b974-e613-4c89-a061-86dd9290f521 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b836dd39-6dd0-4b64-b902-cb810f581070 (dataset): DatasetConfig { id: b836dd39-6dd0-4b64-b902-cb810f581070 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d1a100e-9784-4f32-981f-1a3200e989b4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_d70ae7d1-a48f-47f5-8d1a-20559576d4aa\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, bce22923-b5c7-48f6-9398-4fb24e188e1f (dataset): DatasetConfig { id: bce22923-b5c7-48f6-9398-4fb24e188e1f (dataset), name: DatasetName { pool_name: ZpoolName { id: 0a1d8462-3db8-4b03-acdf-9503499e711d (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_4ee703ec-7859-4e24-b277-ed30a36988ee\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, bf395ec1-38fc-4f51-a537-46e7b1d5ac27 (dataset): DatasetConfig { id: bf395ec1-38fc-4f51-a537-46e7b1d5ac27 (dataset), name: DatasetName { pool_name: ZpoolName { id: 70afe259-ef7a-419a-97ed-388d46a1cb17 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c3ab36e1-bd7a-4bec-8aad-dd5ce5d60764 (dataset): DatasetConfig { id: c3ab36e1-bd7a-4bec-8aad-dd5ce5d60764 (dataset), name: DatasetName { pool_name: ZpoolName { id: 094891fc-36ea-4f74-b7ba-91566beb4534 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_fb142081-54df-4946-b831-3eb44893e9e0\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c4519072-a3de-4af1-8d88-c36f09b720d6 (dataset): DatasetConfig { id: c4519072-a3de-4af1-8d88-c36f09b720d6 (dataset), name: DatasetName { pool_name: ZpoolName { id: 70afe259-ef7a-419a-97ed-388d46a1cb17 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_027e6e69-94cf-42dd-a83a-77f1fe49eb61\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c92f5768-2e8d-43a9-bfc9-1d94e95d1092 (dataset): DatasetConfig { id: c92f5768-2e8d-43a9-bfc9-1d94e95d1092 (dataset), name: DatasetName { pool_name: ZpoolName { id: 094891fc-36ea-4f74-b7ba-91566beb4534 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_9107f0ba-d732-48cb-866e-8ec55c9ba80a\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d135de29-ab84-42f7-96c8-de9249ed8ec5 (dataset): DatasetConfig { id: d135de29-ab84-42f7-96c8-de9249ed8ec5 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, dd142478-5fa8-4853-a153-e41c21a4801b (dataset): DatasetConfig { id: dd142478-5fa8-4853-a153-e41c21a4801b (dataset), name: DatasetName { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_cca5bf73-add8-4186-9f70-0867e2687634\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e0a1e35d-1416-42b2-aa9f-cb29a599ad0f (dataset): DatasetConfig { id: e0a1e35d-1416-42b2-aa9f-cb29a599ad0f (dataset), name: DatasetName { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_63191500-b9e6-45be-85a4-cdf6399d2f2a\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e208e4f4-a40b-422f-9c8f-f1c622569309 (dataset): DatasetConfig { id: e208e4f4-a40b-422f-9c8f-f1c622569309 (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_f84184b7-030d-4a37-95a3-1d6ee7166492\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e6cd7b55-e9f1-437d-958d-3b29fc69c698 (dataset): DatasetConfig { id: e6cd7b55-e9f1-437d-958d-3b29fc69c698 (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e809dea7-039d-4b30-bc75-6586df4b6448 (dataset): DatasetConfig { id: e809dea7-039d-4b30-bc75-6586df4b6448 (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: TransientZone { name: \"oxz_clickhouse_e5ee12e5-569a-47e9-b44f-6d568f6ce98c\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e81ec6aa-db05-4988-829a-241c09c7f9ac (dataset): DatasetConfig { id: e81ec6aa-db05-4988-829a-241c09c7f9ac (dataset), name: DatasetName { pool_name: ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, ecef64a2-124f-47bf-aa94-9f728aff9630 (dataset): DatasetConfig { id: ecef64a2-124f-47bf-aa94-9f728aff9630 (dataset), name: DatasetName { pool_name: ZpoolName { id: f5232c69-2469-4824-a4f8-901936b93f3d (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, eed4c031-32a7-4b3e-a74c-e9a65a249ba7 (dataset): DatasetConfig { id: eed4c031-32a7-4b3e-a74c-e9a65a249ba7 (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_edb48777-436b-4f70-bed2-e6896c7ded23\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ef18a077-6232-49f8-9d2e-980ea45f1c27 (dataset): DatasetConfig { id: ef18a077-6232-49f8-9d2e-980ea45f1c27 (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ef891673-559e-4357-b402-32dc2941fb9c (dataset): DatasetConfig { id: ef891673-559e-4357-b402-32dc2941fb9c (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f1cbcee8-bed9-4295-8534-2a65178413e7 (dataset): DatasetConfig { id: f1cbcee8-bed9-4295-8534-2a65178413e7 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0a1d8462-3db8-4b03-acdf-9503499e711d (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f9e89a5a-3f69-4e57-bca3-f8350c785e78 (dataset): DatasetConfig { id: f9e89a5a-3f69-4e57-bca3-f8350c785e78 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0a1d8462-3db8-4b03-acdf-9503499e711d (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, fb7cfc06-2178-4fbd-9d36-04a0bec5980c (dataset): DatasetConfig { id: fb7cfc06-2178-4fbd-9d36-04a0bec5980c (dataset), name: DatasetName { pool_name: ZpoolName { id: 094891fc-36ea-4f74-b7ba-91566beb4534 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_21413c29-505f-4c82-9c8b-50571ddaefa5\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fffddc0d-7712-4422-ad82-2e7d548c649a (dataset): DatasetConfig { id: fffddc0d-7712-4422-ad82-2e7d548c649a (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d1a100e-9784-4f32-981f-1a3200e989b4 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }} }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.035778243Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:460"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.035918585Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:51820"} {"msg":"datasets ensure","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.036471541Z","hostname":"sock","pid":1566,"sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:859"} {"msg":"Received DatasetsEnsure { config: DatasetsConfig { generation: Generation(1), datasets: {09b0c7ae-601d-42b3-8334-f0bae806ab74 (dataset): DatasetConfig { id: 09b0c7ae-601d-42b3-8334-f0bae806ab74 (dataset), name: DatasetName { pool_name: ZpoolName { id: f5232c69-2469-4824-a4f8-901936b93f3d (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_fa11dff5-c002-4f6a-8b1b-b726a1ca0a20\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0bfe0e66-2d27-46bc-8caf-06cd489a25b5 (dataset): DatasetConfig { id: 0bfe0e66-2d27-46bc-8caf-06cd489a25b5 (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0fa1cc1d-5155-47bd-b94a-71f9b1fb9b84 (dataset): DatasetConfig { id: 0fa1cc1d-5155-47bd-b94a-71f9b1fb9b84 (dataset), name: DatasetName { pool_name: ZpoolName { id: 094891fc-36ea-4f74-b7ba-91566beb4534 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 13e87a70-c620-488f-9c4c-52a3dd4c74b2 (dataset): DatasetConfig { id: 13e87a70-c620-488f-9c4c-52a3dd4c74b2 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 175c6b4c-9871-49c4-9f6d-262d07281ddd (dataset): DatasetConfig { id: 175c6b4c-9871-49c4-9f6d-262d07281ddd (dataset), name: DatasetName { pool_name: ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1c1e6f02-6343-40d6-9459-0ad7ebbe72dd (dataset): DatasetConfig { id: 1c1e6f02-6343-40d6-9459-0ad7ebbe72dd (dataset), name: DatasetName { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1c709747-81e4-4696-a312-d8fc3952bf42 (dataset): DatasetConfig { id: 1c709747-81e4-4696-a312-d8fc3952bf42 (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: TransientZone { name: \"oxz_oximeter_17bd3def-b7fb-4428-a2b5-5f1d745fd5a7\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 25ca6b38-5252-4b35-a88e-c539f69b81e0 (dataset): DatasetConfig { id: 25ca6b38-5252-4b35-a88e-c539f69b81e0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d1a100e-9784-4f32-981f-1a3200e989b4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_9eee15bb-10a1-49cf-8087-610615834db2\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 291f0653-83e9-4b2e-b2ca-f378f3ee968a (dataset): DatasetConfig { id: 291f0653-83e9-4b2e-b2ca-f378f3ee968a (dataset), name: DatasetName { pool_name: ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2d702f49-c939-49fa-a71c-59908195b58e (dataset): DatasetConfig { id: 2d702f49-c939-49fa-a71c-59908195b58e (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_dbcc85a3-d91b-402e-bbe0-02359124c83e\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 359fb39c-3752-4bd5-b0cf-d4273c8e9d01 (dataset): DatasetConfig { id: 359fb39c-3752-4bd5-b0cf-d4273c8e9d01 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 39a7645a-2aa7-401a-9d87-ca92258eea3b (dataset): DatasetConfig { id: 39a7645a-2aa7-401a-9d87-ca92258eea3b (dataset), name: DatasetName { pool_name: ZpoolName { id: 70afe259-ef7a-419a-97ed-388d46a1cb17 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_ae3bb366-dd85-4558-a521-f33b9626f0b1\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3a11fb65-1a6f-46b5-91b2-28f3add03a1a (dataset): DatasetConfig { id: 3a11fb65-1a6f-46b5-91b2-28f3add03a1a (dataset), name: DatasetName { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3f086a5d-8479-4acf-9d3a-f3e97c63a17e (dataset): DatasetConfig { id: 3f086a5d-8479-4acf-9d3a-f3e97c63a17e (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 46a77ce2-fe19-4301-a3df-76f05cf64ad7 (dataset): DatasetConfig { id: 46a77ce2-fe19-4301-a3df-76f05cf64ad7 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0a1d8462-3db8-4b03-acdf-9503499e711d (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4b96cd1d-d3c9-483f-b15d-b2c6178ea9d2 (dataset): DatasetConfig { id: 4b96cd1d-d3c9-483f-b15d-b2c6178ea9d2 (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_0278158b-ebb4-415c-bdce-d9833e816939\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4be48c53-1e9b-472d-b1dd-b0ea92506ee4 (dataset): DatasetConfig { id: 4be48c53-1e9b-472d-b1dd-b0ea92506ee4 (dataset), name: DatasetName { pool_name: ZpoolName { id: a864efea-d3d9-4be0-84a5-f7e73206c783 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 4c1c3fe9-b50d-4e3d-bb08-75de0ef7b512 (dataset): DatasetConfig { id: 4c1c3fe9-b50d-4e3d-bb08-75de0ef7b512 (dataset), name: DatasetName { pool_name: ZpoolName { id: 094891fc-36ea-4f74-b7ba-91566beb4534 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4c40936e-b09e-4620-a87b-09f3378f9031 (dataset): DatasetConfig { id: 4c40936e-b09e-4620-a87b-09f3378f9031 (dataset), name: DatasetName { pool_name: ZpoolName { id: a864efea-d3d9-4be0-84a5-f7e73206c783 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_68e8e65f-6191-43af-b0e4-85cba11d9294\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4fd18126-78b3-4507-b435-97f1efede5d8 (dataset): DatasetConfig { id: 4fd18126-78b3-4507-b435-97f1efede5d8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5344f04e-8cc1-483e-818b-3f158258d437 (dataset): DatasetConfig { id: 5344f04e-8cc1-483e-818b-3f158258d437 (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5c9f7970-3683-4463-9891-b44549db7a6e (dataset): DatasetConfig { id: 5c9f7970-3683-4463-9891-b44549db7a6e (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d1a100e-9784-4f32-981f-1a3200e989b4 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5cae68d6-bc26-4639-ad2d-229af4965de7 (dataset): DatasetConfig { id: 5cae68d6-bc26-4639-ad2d-229af4965de7 (dataset), name: DatasetName { pool_name: ZpoolName { id: a864efea-d3d9-4be0-84a5-f7e73206c783 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_ca441e07-5182-40ca-bf73-5e03b6c846d9\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 695ba87e-8f5d-4571-bb5b-e28635a25d1d (dataset): DatasetConfig { id: 695ba87e-8f5d-4571-bb5b-e28635a25d1d (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6b882b89-3173-44ac-826e-c727dee18043 (dataset): DatasetConfig { id: 6b882b89-3173-44ac-826e-c727dee18043 (dataset), name: DatasetName { pool_name: ZpoolName { id: f5232c69-2469-4824-a4f8-901936b93f3d (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 71175310-7d13-498e-96d4-c720233cf99a (dataset): DatasetConfig { id: 71175310-7d13-498e-96d4-c720233cf99a (dataset), name: DatasetName { pool_name: ZpoolName { id: 70afe259-ef7a-419a-97ed-388d46a1cb17 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 740c3ea5-134c-41a4-a0c2-f625e5be0130 (dataset): DatasetConfig { id: 740c3ea5-134c-41a4-a0c2-f625e5be0130 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 74439e02-29e6-43c9-8771-1221c3e33988 (dataset): DatasetConfig { id: 74439e02-29e6-43c9-8771-1221c3e33988 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 747bdce5-004e-4038-8931-6ce5eaa70188 (dataset): DatasetConfig { id: 747bdce5-004e-4038-8931-6ce5eaa70188 (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 76242899-0849-4deb-a960-ce3e00a06ba8 (dataset): DatasetConfig { id: 76242899-0849-4deb-a960-ce3e00a06ba8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7739caad-267b-467b-8fd5-a50a0096bd4a (dataset): DatasetConfig { id: 7739caad-267b-467b-8fd5-a50a0096bd4a (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d1a100e-9784-4f32-981f-1a3200e989b4 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7ab33682-ac74-48a4-8e88-a54322dbce8c (dataset): DatasetConfig { id: 7ab33682-ac74-48a4-8e88-a54322dbce8c (dataset), name: DatasetName { pool_name: ZpoolName { id: f5232c69-2469-4824-a4f8-901936b93f3d (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 808898e9-4271-442c-8793-5cc9c1376152 (dataset): DatasetConfig { id: 808898e9-4271-442c-8793-5cc9c1376152 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_95fbeccd-695b-4f6e-ae96-35054f4e2171\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 84345852-00ef-4077-a97e-21c8b3f9896f (dataset): DatasetConfig { id: 84345852-00ef-4077-a97e-21c8b3f9896f (dataset), name: DatasetName { pool_name: ZpoolName { id: f5232c69-2469-4824-a4f8-901936b93f3d (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_00711fcf-5ea5-4dbf-918c-e21236b2cac3\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 86147b90-245f-4d55-baee-ea0bb169d87f (dataset): DatasetConfig { id: 86147b90-245f-4d55-baee-ea0bb169d87f (dataset), name: DatasetName { pool_name: ZpoolName { id: f5232c69-2469-4824-a4f8-901936b93f3d (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 86a1286f-398c-4528-b313-5d2eb1b03de4 (dataset): DatasetConfig { id: 86a1286f-398c-4528-b313-5d2eb1b03de4 (dataset), name: DatasetName { pool_name: ZpoolName { id: 094891fc-36ea-4f74-b7ba-91566beb4534 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8701e2b8-78da-46c7-8957-25f526c725ec (dataset): DatasetConfig { id: 8701e2b8-78da-46c7-8957-25f526c725ec (dataset), name: DatasetName { pool_name: ZpoolName { id: a864efea-d3d9-4be0-84a5-f7e73206c783 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8bfdc6cb-8756-4ad3-a327-27f1df997f06 (dataset): DatasetConfig { id: 8bfdc6cb-8756-4ad3-a327-27f1df997f06 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_1877f8e9-a377-4960-91f4-09dd1c711f3b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8c405b05-f6c9-4658-b737-361075b5216f (dataset): DatasetConfig { id: 8c405b05-f6c9-4658-b737-361075b5216f (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d1a100e-9784-4f32-981f-1a3200e989b4 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8c69151f-038e-4a2b-8344-a8aae7b36024 (dataset): DatasetConfig { id: 8c69151f-038e-4a2b-8344-a8aae7b36024 (dataset), name: DatasetName { pool_name: ZpoolName { id: f5232c69-2469-4824-a4f8-901936b93f3d (zpool), kind: External }, kind: TransientZone { name: \"oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 935e6b13-1747-44c8-9f5a-1cb16f0f54a3 (dataset): DatasetConfig { id: 935e6b13-1747-44c8-9f5a-1cb16f0f54a3 (dataset), name: DatasetName { pool_name: ZpoolName { id: a864efea-d3d9-4be0-84a5-f7e73206c783 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9862a669-1af0-4cd7-a424-708337cd7a44 (dataset): DatasetConfig { id: 9862a669-1af0-4cd7-a424-708337cd7a44 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_dbbd09fb-c1a9-46dd-b9f4-782842395cbe\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 998e3b4e-5e5d-4e8c-ad7c-236c106bbd10 (dataset): DatasetConfig { id: 998e3b4e-5e5d-4e8c-ad7c-236c106bbd10 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d1a100e-9784-4f32-981f-1a3200e989b4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_38bf453c-46b6-4fd8-b901-1a35d301feef\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9f6329a2-cbdc-4c02-b4da-15bb972ccf1a (dataset): DatasetConfig { id: 9f6329a2-cbdc-4c02-b4da-15bb972ccf1a (dataset), name: DatasetName { pool_name: ZpoolName { id: 70afe259-ef7a-419a-97ed-388d46a1cb17 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b273b974-e613-4c89-a061-86dd9290f521 (dataset): DatasetConfig { id: b273b974-e613-4c89-a061-86dd9290f521 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b836dd39-6dd0-4b64-b902-cb810f581070 (dataset): DatasetConfig { id: b836dd39-6dd0-4b64-b902-cb810f581070 (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d1a100e-9784-4f32-981f-1a3200e989b4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_d70ae7d1-a48f-47f5-8d1a-20559576d4aa\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, bce22923-b5c7-48f6-9398-4fb24e188e1f (dataset): DatasetConfig { id: bce22923-b5c7-48f6-9398-4fb24e188e1f (dataset), name: DatasetName { pool_name: ZpoolName { id: 0a1d8462-3db8-4b03-acdf-9503499e711d (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_4ee703ec-7859-4e24-b277-ed30a36988ee\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, bf395ec1-38fc-4f51-a537-46e7b1d5ac27 (dataset): DatasetConfig { id: bf395ec1-38fc-4f51-a537-46e7b1d5ac27 (dataset), name: DatasetName { pool_name: ZpoolName { id: 70afe259-ef7a-419a-97ed-388d46a1cb17 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c3ab36e1-bd7a-4bec-8aad-dd5ce5d60764 (dataset): DatasetConfig { id: c3ab36e1-bd7a-4bec-8aad-dd5ce5d60764 (dataset), name: DatasetName { pool_name: ZpoolName { id: 094891fc-36ea-4f74-b7ba-91566beb4534 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_fb142081-54df-4946-b831-3eb44893e9e0\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c4519072-a3de-4af1-8d88-c36f09b720d6 (dataset): DatasetConfig { id: c4519072-a3de-4af1-8d88-c36f09b720d6 (dataset), name: DatasetName { pool_name: ZpoolName { id: 70afe259-ef7a-419a-97ed-388d46a1cb17 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_027e6e69-94cf-42dd-a83a-77f1fe49eb61\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c92f5768-2e8d-43a9-bfc9-1d94e95d1092 (dataset): DatasetConfig { id: c92f5768-2e8d-43a9-bfc9-1d94e95d1092 (dataset), name: DatasetName { pool_name: ZpoolName { id: 094891fc-36ea-4f74-b7ba-91566beb4534 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_9107f0ba-d732-48cb-866e-8ec55c9ba80a\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d135de29-ab84-42f7-96c8-de9249ed8ec5 (dataset): DatasetConfig { id: d135de29-ab84-42f7-96c8-de9249ed8ec5 (dataset), name: DatasetName { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, dd142478-5fa8-4853-a153-e41c21a4801b (dataset): DatasetConfig { id: dd142478-5fa8-4853-a153-e41c21a4801b (dataset), name: DatasetName { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_cca5bf73-add8-4186-9f70-0867e2687634\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e0a1e35d-1416-42b2-aa9f-cb29a599ad0f (dataset): DatasetConfig { id: e0a1e35d-1416-42b2-aa9f-cb29a599ad0f (dataset), name: DatasetName { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_63191500-b9e6-45be-85a4-cdf6399d2f2a\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e208e4f4-a40b-422f-9c8f-f1c622569309 (dataset): DatasetConfig { id: e208e4f4-a40b-422f-9c8f-f1c622569309 (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_f84184b7-030d-4a37-95a3-1d6ee7166492\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e6cd7b55-e9f1-437d-958d-3b29fc69c698 (dataset): DatasetConfig { id: e6cd7b55-e9f1-437d-958d-3b29fc69c698 (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e809dea7-039d-4b30-bc75-6586df4b6448 (dataset): DatasetConfig { id: e809dea7-039d-4b30-bc75-6586df4b6448 (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: TransientZone { name: \"oxz_clickhouse_e5ee12e5-569a-47e9-b44f-6d568f6ce98c\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e81ec6aa-db05-4988-829a-241c09c7f9ac (dataset): DatasetConfig { id: e81ec6aa-db05-4988-829a-241c09c7f9ac (dataset), name: DatasetName { pool_name: ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, ecef64a2-124f-47bf-aa94-9f728aff9630 (dataset): DatasetConfig { id: ecef64a2-124f-47bf-aa94-9f728aff9630 (dataset), name: DatasetName { pool_name: ZpoolName { id: f5232c69-2469-4824-a4f8-901936b93f3d (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, eed4c031-32a7-4b3e-a74c-e9a65a249ba7 (dataset): DatasetConfig { id: eed4c031-32a7-4b3e-a74c-e9a65a249ba7 (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_edb48777-436b-4f70-bed2-e6896c7ded23\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ef18a077-6232-49f8-9d2e-980ea45f1c27 (dataset): DatasetConfig { id: ef18a077-6232-49f8-9d2e-980ea45f1c27 (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ef891673-559e-4357-b402-32dc2941fb9c (dataset): DatasetConfig { id: ef891673-559e-4357-b402-32dc2941fb9c (dataset), name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f1cbcee8-bed9-4295-8534-2a65178413e7 (dataset): DatasetConfig { id: f1cbcee8-bed9-4295-8534-2a65178413e7 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0a1d8462-3db8-4b03-acdf-9503499e711d (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f9e89a5a-3f69-4e57-bca3-f8350c785e78 (dataset): DatasetConfig { id: f9e89a5a-3f69-4e57-bca3-f8350c785e78 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0a1d8462-3db8-4b03-acdf-9503499e711d (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, fb7cfc06-2178-4fbd-9d36-04a0bec5980c (dataset): DatasetConfig { id: fb7cfc06-2178-4fbd-9d36-04a0bec5980c (dataset), name: DatasetName { pool_name: ZpoolName { id: 094891fc-36ea-4f74-b7ba-91566beb4534 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_21413c29-505f-4c82-9c8b-50571ddaefa5\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fffddc0d-7712-4422-ad82-2e7d548c649a (dataset): DatasetConfig { id: fffddc0d-7712-4422-ad82-2e7d548c649a (dataset), name: DatasetName { pool_name: ZpoolName { id: 1d1a100e-9784-4f32-981f-1a3200e989b4 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }} }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.036665824Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/2026aef5-ecc1-428c-aac9-f90b84f99582/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.036671384Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/9404c118-e76c-41a3-b09c-850793e381f3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.036675714Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested datasets', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.036679704Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:909"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.051552981Z","hostname":"sock","pid":1566,"name":"oxp_f5232c69-2469-4824-a4f8-901936b93f3d/crypt/zone/oxz_cockroachdb_fa11dff5-c002-4f6a-8b1b-b726a1ca0a20","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.051593941Z","hostname":"sock","pid":1566,"name":"oxp_f5232c69-2469-4824-a4f8-901936b93f3d/crypt/zone/oxz_cockroachdb_fa11dff5-c002-4f6a-8b1b-b726a1ca0a20","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.051600851Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f5232c69-2469-4824-a4f8-901936b93f3d/crypt/zone/oxz_cockroachdb_fa11dff5-c002-4f6a-8b1b-b726a1ca0a20\"), full_name: \"oxp_f5232c69-2469-4824-a4f8-901936b93f3d/crypt/zone/oxz_cockroachdb_fa11dff5-c002-4f6a-8b1b-b726a1ca0a20\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.114737101Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.114776472Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.114782022Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.164553633Z","hostname":"sock","pid":1566,"name":"oxp_094891fc-36ea-4f74-b7ba-91566beb4534/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.164589464Z","hostname":"sock","pid":1566,"name":"oxp_094891fc-36ea-4f74-b7ba-91566beb4534/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.164594073Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/094891fc-36ea-4f74-b7ba-91566beb4534/crypt/debug\"), full_name: \"oxp_094891fc-36ea-4f74-b7ba-91566beb4534/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.196443022Z","hostname":"sock","pid":1566,"name":"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.196478502Z","hostname":"sock","pid":1566,"name":"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.196483572Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.246653759Z","hostname":"sock","pid":1566,"name":"oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.246685229Z","hostname":"sock","pid":1566,"name":"oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.246689369Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.294105772Z","hostname":"sock","pid":1566,"name":"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.294140883Z","hostname":"sock","pid":1566,"name":"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.294146113Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/8d41776e-a84a-44c7-a848-bc443cc63109/crypt/zone\"), full_name: \"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.323803035Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_oximeter_17bd3def-b7fb-4428-a2b5-5f1d745fd5a7","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.323838285Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_oximeter_17bd3def-b7fb-4428-a2b5-5f1d745fd5a7","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.323843445Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_oximeter_17bd3def-b7fb-4428-a2b5-5f1d745fd5a7\"), full_name: \"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_oximeter_17bd3def-b7fb-4428-a2b5-5f1d745fd5a7\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.380891023Z","hostname":"sock","pid":1566,"name":"oxp_1d1a100e-9784-4f32-981f-1a3200e989b4/crypt/zone/oxz_cockroachdb_9eee15bb-10a1-49cf-8087-610615834db2","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.380927524Z","hostname":"sock","pid":1566,"name":"oxp_1d1a100e-9784-4f32-981f-1a3200e989b4/crypt/zone/oxz_cockroachdb_9eee15bb-10a1-49cf-8087-610615834db2","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.380932594Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1d1a100e-9784-4f32-981f-1a3200e989b4/crypt/zone/oxz_cockroachdb_9eee15bb-10a1-49cf-8087-610615834db2\"), full_name: \"oxp_1d1a100e-9784-4f32-981f-1a3200e989b4/crypt/zone/oxz_cockroachdb_9eee15bb-10a1-49cf-8087-610615834db2\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.442482345Z","hostname":"sock","pid":1566,"name":"oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/zone/oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.442527336Z","hostname":"sock","pid":1566,"name":"oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/zone/oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.442531976Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/zone/oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e\"), full_name: \"oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/zone/oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.501875781Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_cockroachdb_dbcc85a3-d91b-402e-bbe0-02359124c83e","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.501910491Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_cockroachdb_dbcc85a3-d91b-402e-bbe0-02359124c83e","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.501914951Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_cockroachdb_dbcc85a3-d91b-402e-bbe0-02359124c83e\"), full_name: \"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_cockroachdb_dbcc85a3-d91b-402e-bbe0-02359124c83e\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.561005173Z","hostname":"sock","pid":1566,"name":"oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.561043834Z","hostname":"sock","pid":1566,"name":"oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.561048464Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/dfd15883-50bd-419a-82e1-b0ed2f66c5e6)","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:17:49.597192393Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.612382974Z","hostname":"sock","pid":1566,"name":"oxp_70afe259-ef7a-419a-97ed-388d46a1cb17/crypt/zone/oxz_crucible_ae3bb366-dd85-4558-a521-f33b9626f0b1","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.612401744Z","hostname":"sock","pid":1566,"name":"oxp_70afe259-ef7a-419a-97ed-388d46a1cb17/crypt/zone/oxz_crucible_ae3bb366-dd85-4558-a521-f33b9626f0b1","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.612406664Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/70afe259-ef7a-419a-97ed-388d46a1cb17/crypt/zone/oxz_crucible_ae3bb366-dd85-4558-a521-f33b9626f0b1\"), full_name: \"oxp_70afe259-ef7a-419a-97ed-388d46a1cb17/crypt/zone/oxz_crucible_ae3bb366-dd85-4558-a521-f33b9626f0b1\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.673400689Z","hostname":"sock","pid":1566,"name":"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/zone/oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.673435149Z","hostname":"sock","pid":1566,"name":"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/zone/oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.673439429Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/8d41776e-a84a-44c7-a848-bc443cc63109/crypt/zone/oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f\"), full_name: \"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/zone/oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.73241848Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.73245964Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.73246505Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/debug\"), full_name: \"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.761662847Z","hostname":"sock","pid":1566,"name":"oxp_0a1d8462-3db8-4b03-acdf-9503499e711d/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.761701498Z","hostname":"sock","pid":1566,"name":"oxp_0a1d8462-3db8-4b03-acdf-9503499e711d/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.761706788Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_0a1d8462-3db8-4b03-acdf-9503499e711d/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.811724732Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_crucible_0278158b-ebb4-415c-bdce-d9833e816939","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.811756793Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_crucible_0278158b-ebb4-415c-bdce-d9833e816939","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.811760933Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_crucible_0278158b-ebb4-415c-bdce-d9833e816939\"), full_name: \"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_crucible_0278158b-ebb4-415c-bdce-d9833e816939\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.871964648Z","hostname":"sock","pid":1566,"name":"oxp_a864efea-d3d9-4be0-84a5-f7e73206c783/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.872011578Z","hostname":"sock","pid":1566,"name":"oxp_a864efea-d3d9-4be0-84a5-f7e73206c783/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.872016198Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/a864efea-d3d9-4be0-84a5-f7e73206c783/crypt/debug\"), full_name: \"oxp_a864efea-d3d9-4be0-84a5-f7e73206c783/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.901147035Z","hostname":"sock","pid":1566,"name":"oxp_094891fc-36ea-4f74-b7ba-91566beb4534/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.901186915Z","hostname":"sock","pid":1566,"name":"oxp_094891fc-36ea-4f74-b7ba-91566beb4534/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.901191725Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_094891fc-36ea-4f74-b7ba-91566beb4534/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.945772715Z","hostname":"sock","pid":1566,"name":"oxp_a864efea-d3d9-4be0-84a5-f7e73206c783/crypt/zone/oxz_nexus_68e8e65f-6191-43af-b0e4-85cba11d9294","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.945810665Z","hostname":"sock","pid":1566,"name":"oxp_a864efea-d3d9-4be0-84a5-f7e73206c783/crypt/zone/oxz_nexus_68e8e65f-6191-43af-b0e4-85cba11d9294","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:49.945814865Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/a864efea-d3d9-4be0-84a5-f7e73206c783/crypt/zone/oxz_nexus_68e8e65f-6191-43af-b0e4-85cba11d9294\"), full_name: \"oxp_a864efea-d3d9-4be0-84a5-f7e73206c783/crypt/zone/oxz_nexus_68e8e65f-6191-43af-b0e4-85cba11d9294\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.006582836Z","hostname":"sock","pid":1566,"name":"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.006622087Z","hostname":"sock","pid":1566,"name":"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.006627377Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.056628381Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.056660911Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.056664991Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone\"), full_name: \"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.086248363Z","hostname":"sock","pid":1566,"name":"oxp_1d1a100e-9784-4f32-981f-1a3200e989b4/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.086282293Z","hostname":"sock","pid":1566,"name":"oxp_1d1a100e-9784-4f32-981f-1a3200e989b4/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.086287433Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1d1a100e-9784-4f32-981f-1a3200e989b4/crypt/zone\"), full_name: \"oxp_1d1a100e-9784-4f32-981f-1a3200e989b4/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.114098344Z","hostname":"sock","pid":1566,"name":"oxp_a864efea-d3d9-4be0-84a5-f7e73206c783/crypt/zone/oxz_crucible_ca441e07-5182-40ca-bf73-5e03b6c846d9","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.114132954Z","hostname":"sock","pid":1566,"name":"oxp_a864efea-d3d9-4be0-84a5-f7e73206c783/crypt/zone/oxz_crucible_ca441e07-5182-40ca-bf73-5e03b6c846d9","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.114138034Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/a864efea-d3d9-4be0-84a5-f7e73206c783/crypt/zone/oxz_crucible_ca441e07-5182-40ca-bf73-5e03b6c846d9\"), full_name: \"oxp_a864efea-d3d9-4be0-84a5-f7e73206c783/crypt/zone/oxz_crucible_ca441e07-5182-40ca-bf73-5e03b6c846d9\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.174850635Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.174911106Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.174915606Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.224521085Z","hostname":"sock","pid":1566,"name":"oxp_f5232c69-2469-4824-a4f8-901936b93f3d/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.224565796Z","hostname":"sock","pid":1566,"name":"oxp_f5232c69-2469-4824-a4f8-901936b93f3d/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.224572996Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_f5232c69-2469-4824-a4f8-901936b93f3d/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.272572076Z","hostname":"sock","pid":1566,"name":"oxp_70afe259-ef7a-419a-97ed-388d46a1cb17/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.272607157Z","hostname":"sock","pid":1566,"name":"oxp_70afe259-ef7a-419a-97ed-388d46a1cb17/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.272612277Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/70afe259-ef7a-419a-97ed-388d46a1cb17/crypt/debug\"), full_name: \"oxp_70afe259-ef7a-419a-97ed-388d46a1cb17/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.299723589Z","hostname":"sock","pid":1566,"name":"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.29976021Z","hostname":"sock","pid":1566,"name":"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.299765299Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/external_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.346804298Z","hostname":"sock","pid":1566,"name":"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.346840329Z","hostname":"sock","pid":1566,"name":"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.346845569Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.394789119Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.394824229Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.394829529Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/external_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.443028472Z","hostname":"sock","pid":1566,"name":"oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.443064052Z","hostname":"sock","pid":1566,"name":"oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.443080022Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.489070789Z","hostname":"sock","pid":1566,"name":"oxp_1d1a100e-9784-4f32-981f-1a3200e989b4/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.489102369Z","hostname":"sock","pid":1566,"name":"oxp_1d1a100e-9784-4f32-981f-1a3200e989b4/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.489106539Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_1d1a100e-9784-4f32-981f-1a3200e989b4/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.535730923Z","hostname":"sock","pid":1566,"name":"oxp_f5232c69-2469-4824-a4f8-901936b93f3d/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.535773403Z","hostname":"sock","pid":1566,"name":"oxp_f5232c69-2469-4824-a4f8-901936b93f3d/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.535779443Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f5232c69-2469-4824-a4f8-901936b93f3d/crypt/debug\"), full_name: \"oxp_f5232c69-2469-4824-a4f8-901936b93f3d/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.564797828Z","hostname":"sock","pid":1566,"name":"oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/zone/oxz_crucible_95fbeccd-695b-4f6e-ae96-35054f4e2171","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.564829419Z","hostname":"sock","pid":1566,"name":"oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/zone/oxz_crucible_95fbeccd-695b-4f6e-ae96-35054f4e2171","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.564833349Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/zone/oxz_crucible_95fbeccd-695b-4f6e-ae96-35054f4e2171\"), full_name: \"oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/zone/oxz_crucible_95fbeccd-695b-4f6e-ae96-35054f4e2171\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/dfd15883-50bd-419a-82e1-b0ed2f66c5e6)","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:17:50.599365289Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.62301888Z","hostname":"sock","pid":1566,"name":"oxp_f5232c69-2469-4824-a4f8-901936b93f3d/crypt/zone/oxz_crucible_00711fcf-5ea5-4dbf-918c-e21236b2cac3","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.623058231Z","hostname":"sock","pid":1566,"name":"oxp_f5232c69-2469-4824-a4f8-901936b93f3d/crypt/zone/oxz_crucible_00711fcf-5ea5-4dbf-918c-e21236b2cac3","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.623062831Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f5232c69-2469-4824-a4f8-901936b93f3d/crypt/zone/oxz_crucible_00711fcf-5ea5-4dbf-918c-e21236b2cac3\"), full_name: \"oxp_f5232c69-2469-4824-a4f8-901936b93f3d/crypt/zone/oxz_crucible_00711fcf-5ea5-4dbf-918c-e21236b2cac3\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.678955845Z","hostname":"sock","pid":1566,"name":"oxp_f5232c69-2469-4824-a4f8-901936b93f3d/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.678994115Z","hostname":"sock","pid":1566,"name":"oxp_f5232c69-2469-4824-a4f8-901936b93f3d/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.678999305Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f5232c69-2469-4824-a4f8-901936b93f3d/crypt/zone\"), full_name: \"oxp_f5232c69-2469-4824-a4f8-901936b93f3d/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.70717435Z","hostname":"sock","pid":1566,"name":"oxp_094891fc-36ea-4f74-b7ba-91566beb4534/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.7072131Z","hostname":"sock","pid":1566,"name":"oxp_094891fc-36ea-4f74-b7ba-91566beb4534/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.70721775Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/094891fc-36ea-4f74-b7ba-91566beb4534/crypt/zone\"), full_name: \"oxp_094891fc-36ea-4f74-b7ba-91566beb4534/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.737190087Z","hostname":"sock","pid":1566,"name":"oxp_a864efea-d3d9-4be0-84a5-f7e73206c783/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.737222847Z","hostname":"sock","pid":1566,"name":"oxp_a864efea-d3d9-4be0-84a5-f7e73206c783/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.737237037Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/a864efea-d3d9-4be0-84a5-f7e73206c783/crypt/zone\"), full_name: \"oxp_a864efea-d3d9-4be0-84a5-f7e73206c783/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.766398154Z","hostname":"sock","pid":1566,"name":"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/zone/oxz_cockroachdb_1877f8e9-a377-4960-91f4-09dd1c711f3b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.766433124Z","hostname":"sock","pid":1566,"name":"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/zone/oxz_cockroachdb_1877f8e9-a377-4960-91f4-09dd1c711f3b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.766438214Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/8d41776e-a84a-44c7-a848-bc443cc63109/crypt/zone/oxz_cockroachdb_1877f8e9-a377-4960-91f4-09dd1c711f3b\"), full_name: \"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/zone/oxz_cockroachdb_1877f8e9-a377-4960-91f4-09dd1c711f3b\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.824527534Z","hostname":"sock","pid":1566,"name":"oxp_1d1a100e-9784-4f32-981f-1a3200e989b4/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.824561785Z","hostname":"sock","pid":1566,"name":"oxp_1d1a100e-9784-4f32-981f-1a3200e989b4/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.824565985Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_1d1a100e-9784-4f32-981f-1a3200e989b4/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.870852204Z","hostname":"sock","pid":1566,"name":"oxp_f5232c69-2469-4824-a4f8-901936b93f3d/crypt/zone/oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.870885275Z","hostname":"sock","pid":1566,"name":"oxp_f5232c69-2469-4824-a4f8-901936b93f3d/crypt/zone/oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.870889355Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f5232c69-2469-4824-a4f8-901936b93f3d/crypt/zone/oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b\"), full_name: \"oxp_f5232c69-2469-4824-a4f8-901936b93f3d/crypt/zone/oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.928998455Z","hostname":"sock","pid":1566,"name":"oxp_a864efea-d3d9-4be0-84a5-f7e73206c783/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.929033766Z","hostname":"sock","pid":1566,"name":"oxp_a864efea-d3d9-4be0-84a5-f7e73206c783/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.929037936Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_a864efea-d3d9-4be0-84a5-f7e73206c783/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.975613288Z","hostname":"sock","pid":1566,"name":"oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/zone/oxz_cockroachdb_dbbd09fb-c1a9-46dd-b9f4-782842395cbe","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.975663919Z","hostname":"sock","pid":1566,"name":"oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/zone/oxz_cockroachdb_dbbd09fb-c1a9-46dd-b9f4-782842395cbe","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:50.975671799Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/zone/oxz_cockroachdb_dbbd09fb-c1a9-46dd-b9f4-782842395cbe\"), full_name: \"oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/zone/oxz_cockroachdb_dbbd09fb-c1a9-46dd-b9f4-782842395cbe\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.038096241Z","hostname":"sock","pid":1566,"name":"oxp_1d1a100e-9784-4f32-981f-1a3200e989b4/crypt/zone/oxz_crucible_38bf453c-46b6-4fd8-b901-1a35d301feef","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.038132151Z","hostname":"sock","pid":1566,"name":"oxp_1d1a100e-9784-4f32-981f-1a3200e989b4/crypt/zone/oxz_crucible_38bf453c-46b6-4fd8-b901-1a35d301feef","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.038137061Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1d1a100e-9784-4f32-981f-1a3200e989b4/crypt/zone/oxz_crucible_38bf453c-46b6-4fd8-b901-1a35d301feef\"), full_name: \"oxp_1d1a100e-9784-4f32-981f-1a3200e989b4/crypt/zone/oxz_crucible_38bf453c-46b6-4fd8-b901-1a35d301feef\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.095906628Z","hostname":"sock","pid":1566,"name":"oxp_70afe259-ef7a-419a-97ed-388d46a1cb17/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.095962588Z","hostname":"sock","pid":1566,"name":"oxp_70afe259-ef7a-419a-97ed-388d46a1cb17/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.097463726Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_70afe259-ef7a-419a-97ed-388d46a1cb17/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.142968577Z","hostname":"sock","pid":1566,"name":"oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.143002947Z","hostname":"sock","pid":1566,"name":"oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.143007447Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/zone\"), full_name: \"oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.172483978Z","hostname":"sock","pid":1566,"name":"oxp_1d1a100e-9784-4f32-981f-1a3200e989b4/crypt/zone/oxz_crucible_pantry_d70ae7d1-a48f-47f5-8d1a-20559576d4aa","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.172527268Z","hostname":"sock","pid":1566,"name":"oxp_1d1a100e-9784-4f32-981f-1a3200e989b4/crypt/zone/oxz_crucible_pantry_d70ae7d1-a48f-47f5-8d1a-20559576d4aa","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.172532748Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1d1a100e-9784-4f32-981f-1a3200e989b4/crypt/zone/oxz_crucible_pantry_d70ae7d1-a48f-47f5-8d1a-20559576d4aa\"), full_name: \"oxp_1d1a100e-9784-4f32-981f-1a3200e989b4/crypt/zone/oxz_crucible_pantry_d70ae7d1-a48f-47f5-8d1a-20559576d4aa\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.23244248Z","hostname":"sock","pid":1566,"name":"oxp_0a1d8462-3db8-4b03-acdf-9503499e711d/crypt/zone/oxz_crucible_4ee703ec-7859-4e24-b277-ed30a36988ee","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.23248032Z","hostname":"sock","pid":1566,"name":"oxp_0a1d8462-3db8-4b03-acdf-9503499e711d/crypt/zone/oxz_crucible_4ee703ec-7859-4e24-b277-ed30a36988ee","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.23248464Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/0a1d8462-3db8-4b03-acdf-9503499e711d/crypt/zone/oxz_crucible_4ee703ec-7859-4e24-b277-ed30a36988ee\"), full_name: \"oxp_0a1d8462-3db8-4b03-acdf-9503499e711d/crypt/zone/oxz_crucible_4ee703ec-7859-4e24-b277-ed30a36988ee\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.290880284Z","hostname":"sock","pid":1566,"name":"oxp_70afe259-ef7a-419a-97ed-388d46a1cb17/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.290920225Z","hostname":"sock","pid":1566,"name":"oxp_70afe259-ef7a-419a-97ed-388d46a1cb17/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.290925235Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/70afe259-ef7a-419a-97ed-388d46a1cb17/crypt/zone\"), full_name: \"oxp_70afe259-ef7a-419a-97ed-388d46a1cb17/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.320075331Z","hostname":"sock","pid":1566,"name":"oxp_094891fc-36ea-4f74-b7ba-91566beb4534/crypt/zone/oxz_crucible_fb142081-54df-4946-b831-3eb44893e9e0","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.320108161Z","hostname":"sock","pid":1566,"name":"oxp_094891fc-36ea-4f74-b7ba-91566beb4534/crypt/zone/oxz_crucible_fb142081-54df-4946-b831-3eb44893e9e0","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.320113132Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/094891fc-36ea-4f74-b7ba-91566beb4534/crypt/zone/oxz_crucible_fb142081-54df-4946-b831-3eb44893e9e0\"), full_name: \"oxp_094891fc-36ea-4f74-b7ba-91566beb4534/crypt/zone/oxz_crucible_fb142081-54df-4946-b831-3eb44893e9e0\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.377389312Z","hostname":"sock","pid":1566,"name":"oxp_70afe259-ef7a-419a-97ed-388d46a1cb17/crypt/zone/oxz_nexus_027e6e69-94cf-42dd-a83a-77f1fe49eb61","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.377423712Z","hostname":"sock","pid":1566,"name":"oxp_70afe259-ef7a-419a-97ed-388d46a1cb17/crypt/zone/oxz_nexus_027e6e69-94cf-42dd-a83a-77f1fe49eb61","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.377427792Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/70afe259-ef7a-419a-97ed-388d46a1cb17/crypt/zone/oxz_nexus_027e6e69-94cf-42dd-a83a-77f1fe49eb61\"), full_name: \"oxp_70afe259-ef7a-419a-97ed-388d46a1cb17/crypt/zone/oxz_nexus_027e6e69-94cf-42dd-a83a-77f1fe49eb61\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.437344294Z","hostname":"sock","pid":1566,"name":"oxp_094891fc-36ea-4f74-b7ba-91566beb4534/crypt/zone/oxz_crucible_pantry_9107f0ba-d732-48cb-866e-8ec55c9ba80a","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.437387465Z","hostname":"sock","pid":1566,"name":"oxp_094891fc-36ea-4f74-b7ba-91566beb4534/crypt/zone/oxz_crucible_pantry_9107f0ba-d732-48cb-866e-8ec55c9ba80a","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.437391805Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/094891fc-36ea-4f74-b7ba-91566beb4534/crypt/zone/oxz_crucible_pantry_9107f0ba-d732-48cb-866e-8ec55c9ba80a\"), full_name: \"oxp_094891fc-36ea-4f74-b7ba-91566beb4534/crypt/zone/oxz_crucible_pantry_9107f0ba-d732-48cb-866e-8ec55c9ba80a\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.494396022Z","hostname":"sock","pid":1566,"name":"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.494436072Z","hostname":"sock","pid":1566,"name":"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.494441293Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/8d41776e-a84a-44c7-a848-bc443cc63109/crypt/debug\"), full_name: \"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.523761011Z","hostname":"sock","pid":1566,"name":"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/zone/oxz_external_dns_cca5bf73-add8-4186-9f70-0867e2687634","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.523792621Z","hostname":"sock","pid":1566,"name":"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/zone/oxz_external_dns_cca5bf73-add8-4186-9f70-0867e2687634","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.523796871Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/8d41776e-a84a-44c7-a848-bc443cc63109/crypt/zone/oxz_external_dns_cca5bf73-add8-4186-9f70-0867e2687634\"), full_name: \"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/zone/oxz_external_dns_cca5bf73-add8-4186-9f70-0867e2687634\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.585772518Z","hostname":"sock","pid":1566,"name":"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/zone/oxz_crucible_63191500-b9e6-45be-85a4-cdf6399d2f2a","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.585809668Z","hostname":"sock","pid":1566,"name":"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/zone/oxz_crucible_63191500-b9e6-45be-85a4-cdf6399d2f2a","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.585819048Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/8d41776e-a84a-44c7-a848-bc443cc63109/crypt/zone/oxz_crucible_63191500-b9e6-45be-85a4-cdf6399d2f2a\"), full_name: \"oxp_8d41776e-a84a-44c7-a848-bc443cc63109/crypt/zone/oxz_crucible_63191500-b9e6-45be-85a4-cdf6399d2f2a\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/dfd15883-50bd-419a-82e1-b0ed2f66c5e6)","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:17:51.601011859Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.644818889Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_external_dns_f84184b7-030d-4a37-95a3-1d6ee7166492","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.64485592Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_external_dns_f84184b7-030d-4a37-95a3-1d6ee7166492","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.64486043Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_external_dns_f84184b7-030d-4a37-95a3-1d6ee7166492\"), full_name: \"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_external_dns_f84184b7-030d-4a37-95a3-1d6ee7166492\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.70546016Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.70549827Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.70550304Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.755709107Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_clickhouse_e5ee12e5-569a-47e9-b44f-6d568f6ce98c","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.755746517Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_clickhouse_e5ee12e5-569a-47e9-b44f-6d568f6ce98c","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.755763517Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_clickhouse_e5ee12e5-569a-47e9-b44f-6d568f6ce98c\"), full_name: \"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_clickhouse_e5ee12e5-569a-47e9-b44f-6d568f6ce98c\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.816925244Z","hostname":"sock","pid":1566,"name":"oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.816962695Z","hostname":"sock","pid":1566,"name":"oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.816966984Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/debug\"), full_name: \"oxp_3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.844903296Z","hostname":"sock","pid":1566,"name":"oxp_f5232c69-2469-4824-a4f8-901936b93f3d/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.844935857Z","hostname":"sock","pid":1566,"name":"oxp_f5232c69-2469-4824-a4f8-901936b93f3d/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.844939777Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_f5232c69-2469-4824-a4f8-901936b93f3d/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.894239202Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_crucible_pantry_edb48777-436b-4f70-bed2-e6896c7ded23","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.894271233Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_crucible_pantry_edb48777-436b-4f70-bed2-e6896c7ded23","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.894275533Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_crucible_pantry_edb48777-436b-4f70-bed2-e6896c7ded23\"), full_name: \"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_crucible_pantry_edb48777-436b-4f70-bed2-e6896c7ded23\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.95380382Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/clickhouse","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.953860611Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/clickhouse","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:51.953869051Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/clickhouse\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.004133888Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.004174889Z","hostname":"sock","pid":1566,"name":"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.004179239Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293\"), full_name: \"oxp_a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8/crypt/zone/oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.063772117Z","hostname":"sock","pid":1566,"name":"oxp_0a1d8462-3db8-4b03-acdf-9503499e711d/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.063811237Z","hostname":"sock","pid":1566,"name":"oxp_0a1d8462-3db8-4b03-acdf-9503499e711d/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.063816987Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/0a1d8462-3db8-4b03-acdf-9503499e711d/crypt/zone\"), full_name: \"oxp_0a1d8462-3db8-4b03-acdf-9503499e711d/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.09264323Z","hostname":"sock","pid":1566,"name":"oxp_0a1d8462-3db8-4b03-acdf-9503499e711d/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.09267489Z","hostname":"sock","pid":1566,"name":"oxp_0a1d8462-3db8-4b03-acdf-9503499e711d/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.09267908Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/0a1d8462-3db8-4b03-acdf-9503499e711d/crypt/debug\"), full_name: \"oxp_0a1d8462-3db8-4b03-acdf-9503499e711d/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.123384905Z","hostname":"sock","pid":1566,"name":"oxp_094891fc-36ea-4f74-b7ba-91566beb4534/crypt/zone/oxz_nexus_21413c29-505f-4c82-9c8b-50571ddaefa5","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.123420655Z","hostname":"sock","pid":1566,"name":"oxp_094891fc-36ea-4f74-b7ba-91566beb4534/crypt/zone/oxz_nexus_21413c29-505f-4c82-9c8b-50571ddaefa5","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.123425656Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/094891fc-36ea-4f74-b7ba-91566beb4534/crypt/zone/oxz_nexus_21413c29-505f-4c82-9c8b-50571ddaefa5\"), full_name: \"oxp_094891fc-36ea-4f74-b7ba-91566beb4534/crypt/zone/oxz_nexus_21413c29-505f-4c82-9c8b-50571ddaefa5\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.186394953Z","hostname":"sock","pid":1566,"name":"oxp_1d1a100e-9784-4f32-981f-1a3200e989b4/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.186479314Z","hostname":"sock","pid":1566,"name":"oxp_1d1a100e-9784-4f32-981f-1a3200e989b4/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.186602376Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/1d1a100e-9784-4f32-981f-1a3200e989b4/crypt/debug\"), full_name: \"oxp_1d1a100e-9784-4f32-981f-1a3200e989b4/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Writing ledger to /pool/int/2026aef5-ecc1-428c-aac9-f90b84f99582/config/.omicron-datasets.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.21976703Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/9404c118-e76c-41a3-b09c-850793e381f3/config/.omicron-datasets.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.220078184Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"datasets ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.220332857Z","hostname":"sock","pid":1566,"sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:861"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.220475898Z","hostname":"sock","pid":1566,"uri":"/datasets","method":"PUT","req_id":"879a08f8-4b5b-4f2e-8e6c-a40f81291539","remote_addr":"[fd00:1122:3344:101::1]:51820","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3184452,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 63a7090e-2b4b-4dc9-bed6-0a8e375f0293 (service), filesystem_pool: Some(ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0619e98e-96d1-41bb-8d2f-36679c84240f (service), filesystem_pool: Some(ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 69b23a24-f9c6-4e95-90fe-20e66106244e (service), filesystem_pool: Some(ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.2526203Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.252805142Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:59100"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.253122866Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.253128826Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.26352732Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.263573701Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.273044973Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.273104004Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.281694876Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/2026aef5-ecc1-428c-aac9-f90b84f99582/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.281750626Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/9404c118-e76c-41a3-b09c-850793e381f3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.281785387Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.281851948Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.32582965Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.32586139Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.325867041Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.37634006Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.379993784Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.383756209Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.396920965Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.401026834Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:52.404978981Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/dfd15883-50bd-419a-82e1-b0ed2f66c5e6)","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:17:52.602944582Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Profile for oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.050956635Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.051052196Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.282931871Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4467"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.282987971Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4494"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:17:54.312494452Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5300053050309105182"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:17:54.312610413Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9777168582998979993"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.321848843Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4507"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.321894254Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4353"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.371572994Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4395"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.371615534Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4474","rack_subnet":"fd00:1122:3344::"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.390699191Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4485"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.390734581Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4527"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.457295872Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4559"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.457329463Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4565"} {"msg":"Profile for oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.533950843Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.543097922Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4606"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.543162372Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:370"} {"msg":"Failed to get switch ID from MGS (retrying in 51.924638ms)","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:54.683267787Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:389","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)"} {"msg":"Failed to get switch ID from MGS (retrying in 136.807669ms)","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:54.736492689Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:389","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.874332067Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:420"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.906629551Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:717"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.906732482Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:762"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.907182447Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:765","uptime":"0"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.907210178Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:444","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: Some(\n None,\n ),\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n tx_eq: None,\n },\n },\n },\n}"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.943464139Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4145"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.992541501Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.992596771Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.992604361Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.992623591Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.992630972Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.992637132Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.992642502Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.992649182Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.992672602Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.992678472Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.992684632Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.992690772Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.992696163Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.992700343Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.992704423Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.992708512Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.992712643Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.992716693Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.992720753Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.992724783Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.992732453Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.992736693Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:54.992740923Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:55.015085929Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:55.01517233Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:55.01526058Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:55.019795084Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"Booting oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:55.019849065Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:17:55.024059795Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293\") failed: Property not found. retry in 71.387042ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:55.152183337Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f\") failed: Property not found. retry in 39.143072ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:55.157947176Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e\") failed: Property not found. retry in 59.478572ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:55.163511572Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f\") failed: Property not found. retry in 95.925284ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:55.203816981Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e\") failed: Property not found. retry in 93.61745ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:55.324747108Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293\") failed: Property not found. retry in 53.340202ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:55.350985419Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f\") failed: Property not found. retry in 268.38658ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:55.367828699Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293\") failed: Property not found. retry in 254.098503ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:55.477880017Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e\") failed: Property not found. retry in 259.45305ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:55.515310832Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f\") failed: Property not found. retry in 428.439545ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:55.654451735Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293\") failed: Property not found. retry in 512.6165ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:55.751002052Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e\") failed: Property not found. retry in 463.185535ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:55.799375516Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:17:55.985347905Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:17:55.985478227Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"wait for service Some(\"oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f\") failed: Property not found. retry in 908.588386ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:56.09347321Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e\") failed: Property not found. retry in 496.588706ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:56.273122054Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293\") failed: Property not found. retry in 546.552527ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:56.282722428Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e\") failed: Property not found. retry in 626.916612ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:56.779188367Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293\") failed: Property not found. retry in 891.439697ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:56.839150349Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f\") failed: Property not found. retry in 1.29816112s","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:57.012328196Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e\") failed: Property not found. retry in 760.355194ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:57.415940131Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293\") failed: Property not found. retry in 1.265081531s","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:57.741380478Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e\") failed: Property not found. retry in 940.376177ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:58.185517174Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f\") failed: Property not found. retry in 517.438896ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:58.319466335Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f\") failed: Property not found. retry in 819.25562ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:58.845600517Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293\") failed: Property not found. retry in 548.397307ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:59.013824015Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e\") failed: Property not found. retry in 567.221787ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:59.134726811Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293\") failed: Property not found. retry in 1.457004548s","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:59.572059207Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f\") failed: Property not found. retry in 510.255534ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:59.674021208Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e\") failed: Property not found. retry in 1.147763055s","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:17:59.711285771Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f\") failed: Property not found. retry in 558.186106ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:00.193461899Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.051213149Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_internal_dns_0619e98e-96d1-41bb-8d2f-36679c84240f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.051266309Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_internal_dns_69b23a24-f9c6-4e95-90fe-20e66106244e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.0512808Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_internal_dns_63a7090e-2b4b-4dc9-bed6-0a8e375f0293"} {"msg":"Writing ledger to /pool/int/2026aef5-ecc1-428c-aac9-f90b84f99582/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.0512925Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/9404c118-e76c-41a3-b09c-850793e381f3/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.051625953Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.051916887Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"612efa53-7657-4b0f-b0de-34d971172dde","remote_addr":"[fd00:1122:3344:101::1]:59100","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8798981,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.05382166Z","hostname":"sock","pid":1566,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:658"} {"msg":"failed to write DNS configuration (will retry in 135.763792ms)","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:01.63628211Z","hostname":"sock","pid":1566,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:678","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): client error (Connect): tcp connect error: Connection refused (os error 146): Connection refused (os error 146)"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.792931551Z","hostname":"sock","pid":1566,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:658"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.839885489Z","hostname":"sock","pid":1566,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:658"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.887530835Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:694"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.887546295Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.920019081Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.9207576Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.92076622Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 63a7090e-2b4b-4dc9-bed6-0a8e375f0293 (service), filesystem_pool: Some(ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0619e98e-96d1-41bb-8d2f-36679c84240f (service), filesystem_pool: Some(ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 69b23a24-f9c6-4e95-90fe-20e66106244e (service), filesystem_pool: Some(ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 699de3c0-293d-43c3-8434-a57a6e89759b (service), filesystem_pool: Some(ZpoolName { id: f5232c69-2469-4824-a4f8-901936b93f3d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 38ad58e5-0e2c-44c0-be50-be45adea57c9, kind: Service { id: 699de3c0-293d-43c3-8434-a57a6e89759b }, name: Name(\"ntp-699de3c0-293d-43c3-8434-a57a6e89759b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 92])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.950453932Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.950519873Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:47888"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.950720846Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.950730616Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: a3b0f374-f5e0-4dee-9a37-ffb49c3bc4e8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.962695678Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.962749218Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d41776e-a84a-44c7-a848-bc443cc63109 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.97295112Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.9729937Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 3f7bc1a3-3a0a-44ab-a7b2-f71b8c7f753b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.982174029Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/2026aef5-ecc1-428c-aac9-f90b84f99582/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.98222063Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/9404c118-e76c-41a3-b09c-850793e381f3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.982420782Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.982555094Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:01.993545314Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:02.02599269Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:02.027657449Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:02.02766284Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:02.057449203Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 229, 92]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:02.057464204Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1275","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:02.057932049Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:85"} {"msg":"creating nat entry for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:02.057940519Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:92"} {"msg":"creation of nat entry successful for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:02.058196343Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:120"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:02.058884731Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:02.08242747Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:02.101689419Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:02.725306138Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:02.7254572Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b\") failed: Property not found. retry in 59.590134ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:02.861432566Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b\") failed: Property not found. retry in 132.202838ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:02.932228457Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b\") failed: Property not found. retry in 203.017483ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:03.076070105Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b\") failed: Property not found. retry in 313.307ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:03.288471598Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b\") failed: Property not found. retry in 604.034196ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:03.610365873Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b\") failed: Property not found. retry in 502.065767ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:04.224232585Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:04.31556013Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5300053050309105182"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:04.315673162Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9777168582998979993"} {"msg":"wait for service Some(\"oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b\") failed: Property not found. retry in 681.403709ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:04.735837763Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b\") failed: Property not found. retry in 547.114092ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:05.426218465Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b\") failed: Property not found. retry in 1.495711008s","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:05.982069558Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:05.9872632Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:05.987394882Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"wait for service Some(\"oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b\") failed: Property not found. retry in 1.206429017s","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:07.487777957Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:08.713304017Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3336","zone":"oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b"} {"msg":"Writing ledger to /pool/int/2026aef5-ecc1-428c-aac9-f90b84f99582/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:08.713393898Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/9404c118-e76c-41a3-b09c-850793e381f3/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:08.713820123Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:08.714177907Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"d1deea3f-2974-4316-ac8e-94c4e6d661a1","remote_addr":"[fd00:1122:3344:101::1]:47888","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6763577,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:08.716831079Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:735"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:08.747716646Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:08.747871298Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:33166"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:08.75645398Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3926"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:08.756624931Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"a1f20e27-3c0b-46e0-94b1-203b95a88536","remote_addr":"[fd00:1122:3344:101::1]:33166","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8560,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:08.758366132Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:08.856775212Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:08.856936733Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:53385"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:10.777020195Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"16432527369878710882"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:10.885341512Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15596257807682548297"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:11.05362209Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"8597029270196437948"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:12.865658698Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3926"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:12.8657829Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"d382c41a-036a-42cd-b609-bdbd18f874f6","remote_addr":"[fd00:1122:3344:101::1]:53385","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008777,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:12.868149188Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:13.021519849Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:13.021687701Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:36732"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:13.030141841Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3926"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:13.030257323Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"6f6e8b95-8fea-4abf-bd9c-f303919e9aef","remote_addr":"[fd00:1122:3344:101::1]:36732","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8500,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:13.032629671Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:13.26489395Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:13.265029272Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:44152"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_699de3c0-293d-43c3-8434-a57a6e89759b': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:13.273561653Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3926"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:13.273683775Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"e4eceaa0-a740-4b48-ab86-4b40dc803ab7","remote_addr":"[fd00:1122:3344:101::1]:44152","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8551,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:13.27665695Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:13.52578984Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:13.525894321Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:48775"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:13.534320782Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"fc2e94be-ed79-4048-b773-dffcff10c2ca","remote_addr":"[fd00:1122:3344:101::1]:48775","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8384,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:13.537482779Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:13.537491099Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:13.691947414Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:13.692049725Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:64373"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:13.699451823Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"f22b17d6-af87-4414-b23b-355c15c57bd1","remote_addr":"[fd00:1122:3344:101::1]:64373","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7350,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:13.700977262Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:13.700982591Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:14.098411983Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:14.098566675Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:56540"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:14.10907495Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"3aec9976-da6d-40f8-9a13-2dd3957f5ef8","remote_addr":"[fd00:1122:3344:101::1]:56540","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":10406,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:14.112753834Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:14.112759774Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:14.318205875Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5300053050309105182"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:14.318325546Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9777168582998979993"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:14.396844699Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:14.39694637Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:54187"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:14.404849944Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"aaa65c03-e6b0-4cc1-a49a-bf06d4333e76","remote_addr":"[fd00:1122:3344:101::1]:54187","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7811,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:14.406769097Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:14.406776407Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:14.753050881Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:14.753186383Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:37660"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:14.762014028Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"15555fc5-4759-4a73-96a8-c809a684bffa","remote_addr":"[fd00:1122:3344:101::1]:37660","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8751,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:14.763534195Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:14.763539286Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:15.437733675Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:15.437886177Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:35325"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:15.447277968Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"b63c6651-aee5-4688-aa09-3f0121b4fa3c","remote_addr":"[fd00:1122:3344:101::1]:35325","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9332,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:15.450008371Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:15.450016331Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:15.998526937Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:15.998705009Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:40891"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:16.004102523Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:16.004274235Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:16.008327233Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"83cb7f40-1b28-43e4-82af-ace559676e58","remote_addr":"[fd00:1122:3344:101::1]:40891","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9537,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:16.011761064Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:16.011771934Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:16.637197095Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:16.637334126Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:60325"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:16.645465273Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"9ec0b2ce-2e29-42f8-90c1-efe083ec9ff7","remote_addr":"[fd00:1122:3344:101::1]:60325","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8066,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:16.647508217Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:16.647517887Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:17.732386536Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:17.732534057Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:40435"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:17.741446203Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"9b410e83-54a0-4450-ae32-43704ea7a4ee","remote_addr":"[fd00:1122:3344:101::1]:40435","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8826,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:17.743478497Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:17.743485948Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:18.716903752Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2768020810680673949"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:18.717018493Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12050495470882410828"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:19.028349082Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:19.028479763Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:35497"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:19.037157466Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"fbd06fbb-5379-4f60-8102-8dfdf85f7468","remote_addr":"[fd00:1122:3344:101::1]:35497","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8554,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:19.041017292Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:19.041028023Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:19.38423481Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"256.481874ms"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:20.246127939Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:20.246256301Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:52019"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:20.255312508Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"e42aa776-86b2-46a4-a9dd-dd27f4b86b97","remote_addr":"[fd00:1122:3344:101::1]:52019","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9003,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:20.257571275Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:20.257583835Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:20.778854318Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"16432527369878710882"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:20.886149783Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15596257807682548297"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:21.055468054Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"8597029270196437948"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:21.467765822Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:21.467940095Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:57308"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:21.476970372Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"f359405d-5236-4df1-8098-feb151379d56","remote_addr":"[fd00:1122:3344:101::1]:57308","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8934,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:21.478810944Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:21.478817164Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:22.628695385Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:22.628898697Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:46161"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:22.637385058Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"12d7d0df-2141-4b14-8aae-6676e5e55b78","remote_addr":"[fd00:1122:3344:101::1]:46161","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8393,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:22.639798446Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:22.639806317Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:24.250390001Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:24.250538713Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:42701"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:24.258990513Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"5e0ab79c-8788-4357-88b9-42263ff26974","remote_addr":"[fd00:1122:3344:101::1]:42701","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8382,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:24.263034281Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:24.263042511Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:24.320488474Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5300053050309105182"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:24.320610905Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9777168582998979993"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:24.890096891Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:44237"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:24.890323424Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"77370905-3a0d-460d-8151-ed025c9aa8be","remote_addr":"[fdb0:200::2]:44237","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":76,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:26.006448023Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:26.006563734Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:26.135278164Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:26.135442525Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:36791"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:26.144959328Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"8f438514-e62e-476c-a299-8908aeb95f54","remote_addr":"[fd00:1122:3344:101::1]:36791","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9450,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:26.148820044Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:26.148827954Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:28.71908562Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2768020810680673949"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:28.719204321Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12050495470882410828"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:29.647919185Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:29.648079097Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:48388"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:29.656640559Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"3b1089d9-99e0-4995-a06c-e94ca4e18c72","remote_addr":"[fd00:1122:3344:101::1]:48388","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8509,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:29.660744287Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:29.660752037Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:30.780957516Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"16432527369878710882"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:30.887130307Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15596257807682548297"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:31.057326579Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"8597029270196437948"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:32.765501933Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:32.765682165Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:62808"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:32.775021626Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"8e7aab84-644f-47e9-8f94-71ebedcbd874","remote_addr":"[fd00:1122:3344:101::1]:62808","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9235,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:32.777224322Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:32.777231742Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:34.323373581Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5300053050309105182"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:34.323615434Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9777168582998979993"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:34.67514296Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"564.763778ms"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:36.008794514Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:36.008909495Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:36.139596548Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:36.139741069Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:39014"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:36.148807877Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"4b7898b5-ee28-4851-86f7-c3b2ffbe565e","remote_addr":"[fd00:1122:3344:101::1]:39014","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8952,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:36.1532563Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:36.15326448Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:38.721377001Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2768020810680673949"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:38.721489442Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12050495470882410828"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:39.848250348Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:39.84839527Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:46480"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:39.856987182Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"0c894136-cf06-44c0-aa6a-f8fd8aed6ebd","remote_addr":"[fd00:1122:3344:101::1]:46480","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8516,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:39.859642583Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:39.859655554Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:40.783095484Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"16432527369878710882"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:40.888371345Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15596257807682548297"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:41.057710516Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"8597029270196437948"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:43.034177617Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:43.034382199Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:33372"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:43.043568519Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"899945ca-ab81-4ae9-a20d-1294d575cc51","remote_addr":"[fd00:1122:3344:101::1]:33372","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9088,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:43.045830286Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:43.045837815Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:44.32480911Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"5300053050309105182"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:44.325376927Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"9777168582998979993"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:46.011137054Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:46.011243925Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:48.723018872Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2768020810680673949"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:48.723253465Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12050495470882410828"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:48.9930697Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:48.993259502Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:58638"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:49.001882154Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"6efb64b0-15b3-4751-8787-7b4fbcf8814f","remote_addr":"[fd00:1122:3344:101::1]:58638","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8536,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:49.004384314Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:49.004392564Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:50.277476079Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"1.069473735s"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:50.784910818Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"16432527369878710882"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:50.889206667Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15596257807682548297"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:51.05951235Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"8597029270196437948"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:54.925390388Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:36063"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:54.92549727Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"12f4972d-9a60-4465-8af0-df3eaad33438","remote_addr":"[fdb0:200::2]:36063","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":46,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:56.013095269Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:56.013203511Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:57.395460423Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:57.395611754Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:50224"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:57.404581921Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"065caaa6-1a5a-48de-bece-481b60f38b5e","remote_addr":"[fd00:1122:3344:101::1]:50224","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8905,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:18:57.407389034Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:18:57.407397625Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:58.725341912Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2768020810680673949"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:18:58.725455293Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12050495470882410828"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:19:00.787045536Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"16432527369878710882"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:19:00.890538806Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"15596257807682548297"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:19:01.06093828Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"8597029270196437948"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:19:06.015189237Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:19:06.015337859Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:19:06.382911376Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"2.79050156s"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:19:08.726677412Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"2768020810680673949"} {"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:19:08.726913194Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"12050495470882410828"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:19:09.901020073Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:19:09.901191575Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:63317"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:19:09.910447205Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"f90653bc-62be-4f3b-9eaa-eaa4c1037509","remote_addr":"[fd00:1122:3344:101::1]:63317","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9170,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:19:09.913068706Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:19:09.913076686Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:19:16.016843329Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:19:16.01694887Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:19:24.207406395Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"5.74934884s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:19:24.959128956Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:59415"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:19:24.959215987Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"b619a42c-2704-42a4-aa5b-fc4e98d4e3f1","remote_addr":"[fdb0:200::2]:59415","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":31,"response_code":"200"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:19:25.762695922Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:19:25.762822834Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:64628"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:19:25.771787721Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"4ce76e3b-8cc4-4b5f-a45a-7718c3162d5d","remote_addr":"[fd00:1122:3344:101::1]:64628","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8908,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:19:25.774390552Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:19:25.774399701Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:19:26.018625182Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:19:26.018747874Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:19:36.020883782Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:19:36.021005413Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:19:44.990983338Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"4.613578865s"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:19:46.009088364Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:19:46.009257106Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:57175"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:19:46.017873058Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"8f45fa6d-d3aa-448d-a7d3-3a958a2378fa","remote_addr":"[fd00:1122:3344:101::1]:57175","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8523,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:19:46.020274487Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:19:46.020283847Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:19:46.023407514Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:19:46.023526476Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:19:54.993942286Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:45490"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:19:54.994061988Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"6cff8169-46a7-4ca2-9607-00e49c08a26d","remote_addr":"[fdb0:200::2]:45490","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":44,"response_code":"200"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:19:55.935609564Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:19:55.935722416Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:63208"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:19:55.945588783Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"65f568b2-0f7f-4a22-9ab7-ef275cfd0924","remote_addr":"[fd00:1122:3344:101::1]:63208","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9757,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:19:55.948148123Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:19:55.948159143Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:19:56.025199187Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:19:56.025342639Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:20:04.641851226Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"16.790021411s"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:20:06.026624177Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:20:06.026735839Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:20:16.029439543Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:20:16.029540084Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:20:17.283499662Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:20:17.283634624Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:59985"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:20:17.292482439Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"05da00d3-ac87-463c-82a2-4d1be77d2206","remote_addr":"[fd00:1122:3344:101::1]:59985","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8780,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:20:17.296831341Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:20:17.296850051Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:20:25.026094525Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:41293"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:20:25.026171246Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"4a62192d-8312-46df-b761-161c2ce262d9","remote_addr":"[fdb0:200::2]:41293","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":35,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:20:26.031245237Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:20:26.031420239Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:20:34.157661701Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:20:34.157807173Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:43360"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:20:34.167334246Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"43411580-f838-4ceb-b3c3-fc0b10f8b4ec","remote_addr":"[fd00:1122:3344:101::1]:43360","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9467,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:20:34.172164223Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:20:34.172175854Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:20:36.033534388Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:20:36.033645299Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:20:36.466671454Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"34.045936274s"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:20:46.03603981Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:20:46.036127311Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:20:55.059492212Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:50919"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:20:55.059633514Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"0efd8e27-3a63-4922-8e54-2a2283be41de","remote_addr":"[fdb0:200::2]:50919","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":60,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:20:56.037851276Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:20:56.037954587Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:20:56.785670331Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:20:56.785804842Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:37765"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:20:56.794379804Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"f13791f7-532d-4ef7-9259-987ff50ecf51","remote_addr":"[fd00:1122:3344:101::1]:37765","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8521,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:20:56.799210361Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:20:56.799221432Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:21:06.03969131Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:21:06.039818881Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:21:16.041663934Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:21:16.041774636Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:21:25.09244686Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:39422"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:21:25.092559291Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"545696f8-2880-4dd7-9edc-fe27ca6f2640","remote_addr":"[fdb0:200::2]:39422","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":45,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:21:25.547889191Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"56.626711824s"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:21:26.044173437Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:21:26.04442764Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:21:36.046505987Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:21:36.046630728Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:21:43.328445549Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:21:43.32858409Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:37336"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:21:43.337715459Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"ff2a787e-dfb4-4aa7-ab3b-dc3b17ef454a","remote_addr":"[fd00:1122:3344:101::1]:37336","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9064,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:21:43.342802399Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:21:43.342815179Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:21:46.049314422Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:21:46.049419933Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:21:55.124680498Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:41111"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:21:55.124761739Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"a373afa3-5eaa-4d6d-866b-d6c2f1fa86b4","remote_addr":"[fdb0:200::2]:41111","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":36,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:21:56.050520788Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:21:56.050626629Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:22:06.053259753Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:22:06.053360794Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:22:16.054679241Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:22:16.054829523Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:22:25.158201443Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:59060"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:22:25.158289165Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"c7190b08-424f-4cc3-b65c-0c3c5c939307","remote_addr":"[fdb0:200::2]:59060","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":39,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:22:26.057233804Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:22:26.057343235Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:22:26.427847597Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:22:26.428018799Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:39115"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:22:26.43736896Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"829fbccd-2345-4c13-b16f-88b0ff552bd0","remote_addr":"[fd00:1122:3344:101::1]:39115","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9245,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:22:26.440142593Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:22:26.440155264Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:22:36.059044898Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:22:36.05923735Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:22:37.209157332Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"94.939681943s"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:22:46.061030714Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:22:46.061276607Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:22:49.004907539Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:22:55.192205466Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:54968"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:22:55.192326837Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"58d9c6a1-174c-4f19-8623-5b082ac86b57","remote_addr":"[fdb0:200::2]:54968","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":46,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:22:56.062756398Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:22:56.062867559Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:22:59.174219133Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:22:59.174350315Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:51029"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:22:59.183646055Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"d94750c0-cd9e-4cd3-992d-5e5da67a3f23","remote_addr":"[fd00:1122:3344:101::1]:51029","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9188,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:22:59.188514493Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:22:59.188523323Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:23:06.06535821Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:23:06.065601884Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:23:16.067149956Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:23:16.067261327Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:23:25.223537936Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:33875"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:23:25.223658197Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"95abd020-28c0-400f-9af4-2f3f94ff636e","remote_addr":"[fdb0:200::2]:33875","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":46,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:23:26.069061392Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:23:26.069179113Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:23:35.215455083Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:23:35.215557554Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:49147"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:23:35.224139296Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"8e5d33ae-3e27-4233-a4eb-921da436c045","remote_addr":"[fd00:1122:3344:101::1]:49147","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8489,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:23:35.226790508Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:23:35.226798957Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:23:36.071635934Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:23:36.071744505Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:23:46.073954874Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:23:46.074058125Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:23:55.258364056Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:61219"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:23:55.258503217Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"e774fa2d-1a31-4c10-9606-a962f5fce045","remote_addr":"[fdb0:200::2]:61219","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":52,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:23:56.075740957Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:23:56.07597157Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:24:06.078150037Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:24:06.078244309Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:24:16.080062893Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:24:16.080302196Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:24:19.112102535Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:24:19.112306427Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:61657"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:24:19.121143543Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"079a6cf1-7a11-471e-adcb-039bba2b27de","remote_addr":"[fd00:1122:3344:101::1]:61657","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8714,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:24:19.124217779Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:24:19.124226229Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:24:25.290918242Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:35953"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:24:25.291038363Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"18ad84be-077f-4af2-b0a3-a43a9abf05bb","remote_addr":"[fdb0:200::2]:35953","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":46,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:24:26.081926288Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:24:26.08204208Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:24:27.186149247Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"223.236929212s"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:24:36.084234829Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:24:36.084476112Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:24:46.086111303Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:24:46.086218115Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:24:55.323494846Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:44710"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:24:55.323572737Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"0f4b7003-88ce-4810-924d-d9f42af27e42","remote_addr":"[fdb0:200::2]:44710","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":36,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:24:56.088190971Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:24:56.088290012Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:25:06.090190958Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:25:06.090295139Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:25:16.092167373Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:25:16.092402656Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:25:25.356082151Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:44639"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:25:25.356161072Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"34dba5d5-22d7-4c36-8eb1-9048defb3e77","remote_addr":"[fdb0:200::2]:44639","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":36,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:25:26.093605543Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:25:26.093713724Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:25:27.217771269Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:25:27.21788766Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:42727"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:25:27.22712122Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"fea5d7c6-93fb-44cf-a331-eba22b06d444","remote_addr":"[fd00:1122:3344:101::1]:42727","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9066,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:25:27.229935163Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-11T01:25:27.229945673Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:25:36.095945754Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:25:36.096157276Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:25:46.098316614Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:25:46.098426456Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:25:55.389764151Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:54487"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:25:55.389896112Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"1e1ed818-a6ff-4cc9-a87f-079b81084be3","remote_addr":"[fdb0:200::2]:54487","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":47,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:25:56.100458453Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:25:56.100618206Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:26:06.10324912Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:26:06.103482332Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:26:16.105111573Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:26:16.105218574Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:26:25.423633289Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:45774"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:26:25.423739921Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"03043d64-1010-4979-bb9d-4dfd8e483ae0","remote_addr":"[fdb0:200::2]:45774","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":39,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:26:26.107240371Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:26:26.107351313Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:26:36.108421768Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:26:36.108517779Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:26:46.110333783Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:26:46.110446274Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:26:55.456543267Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:40135"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-11T01:26:55.456620718Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"f7078848-fb9f-43c0-9b07-3b9570819e45","remote_addr":"[fdb0:200::2]:40135","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":37,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:26:56.111971434Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4153943977543823991"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-11T01:26:56.112085116Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"dfd15883-50bd-419a-82e1-b0ed2f66c5e6","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2612706593396418549"}