[ Nov 21 23:09:58 Enabled. ]
[ Nov 21 23:09:58 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ]
[ Nov 21 23:09:58 Method "start" exited with status 0. ]
[ Nov 21 23:09:58 Rereading configuration. ]
[ Nov 21 23:09:58 No 'refresh' method defined.  Treating as :true. ]
note: configured to log to "/dev/stdout"
{"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:09:58.825625877Z","hostname":"sock","pid":1566,"file":"illumos-utils/src/running_zone.rs:113"}
{"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:09:58.825788839Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"}
{"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:09:58.877739681Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"}
{"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:09:58.877781911Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"}
{"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:09:58.993458859Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"}
{"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:09:59.003124542Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"}
{"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:09:59.011163249Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:09:59.608263949Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:200::, width: 64 }"}
{"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:09:59.743230673Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:126"}
{"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:09:59.743263473Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:138"}
{"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:09:59.743371905Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:151"}
{"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:09:59.743458267Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"}
{"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:09:59.743483477Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"}
{"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:09:59.743495807Z","hostname":"sock","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:717"}
{"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:09:59.746407213Z","hostname":"sock","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:624"}
{"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:09:59.746414623Z","hostname":"sock","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:766"}
{"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:09:59.746426613Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:187"}
{"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:09:59.746430194Z","hostname":"sock","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:825"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:09:59.746452854Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"}
{"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:09:59.746498824Z","hostname":"sock","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:09:59.746588706Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:09:59.746610736Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:09:59.746620066Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:09:59.758580795Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"}
{"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:09:59.758607946Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"67a39b8e-1dc3-40e2-8b0f-59c429bf4f91 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:09:59.858455914Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 67a39b8e-1dc3-40e2-8b0f-59c429bf4f91 (zpool), kind: Internal }"}
{"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.339477879Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.339520139Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.339527059Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.33954138Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.33955316Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.33955937Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.33958053Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.348314458Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"}
{"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.348335809Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"4c94aa86-6837-420b-a3c1-83899731e9b0 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.436262669Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 4c94aa86-6837-420b-a3c1-83899731e9b0 (zpool), kind: Internal }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.92212012Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922165041Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 67a39b8e-1dc3-40e2-8b0f-59c429bf4f91 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4c94aa86-6837-420b-a3c1-83899731e9b0 (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922201901Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922215072Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922224052Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922229182Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922235512Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922247882Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 67a39b8e-1dc3-40e2-8b0f-59c429bf4f91 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4c94aa86-6837-420b-a3c1-83899731e9b0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922264062Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922268372Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922272303Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922288503Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922292243Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922302073Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 67a39b8e-1dc3-40e2-8b0f-59c429bf4f91 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4c94aa86-6837-420b-a3c1-83899731e9b0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922319723Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922323753Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922329133Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922334953Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922338384Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922341744Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 67a39b8e-1dc3-40e2-8b0f-59c429bf4f91 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4c94aa86-6837-420b-a3c1-83899731e9b0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922356824Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922370374Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922374284Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922377644Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922382084Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922385534Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 67a39b8e-1dc3-40e2-8b0f-59c429bf4f91 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4c94aa86-6837-420b-a3c1-83899731e9b0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922404255Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922409075Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922413245Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922417465Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922454075Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922467026Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922473646Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 67a39b8e-1dc3-40e2-8b0f-59c429bf4f91 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4c94aa86-6837-420b-a3c1-83899731e9b0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922503106Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922510186Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922514656Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922518016Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922522256Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922528427Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922532477Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 67a39b8e-1dc3-40e2-8b0f-59c429bf4f91 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4c94aa86-6837-420b-a3c1-83899731e9b0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922561977Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922566637Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922570107Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922574837Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922579147Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922583188Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 67a39b8e-1dc3-40e2-8b0f-59c429bf4f91 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4c94aa86-6837-420b-a3c1-83899731e9b0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922624308Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922629488Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922633108Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922636858Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 67a39b8e-1dc3-40e2-8b0f-59c429bf4f91 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4c94aa86-6837-420b-a3c1-83899731e9b0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922665059Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922670499Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922675379Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922678659Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922686159Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:97"}
{"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922697409Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:99"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922702279Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 67a39b8e-1dc3-40e2-8b0f-59c429bf4f91 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4c94aa86-6837-420b-a3c1-83899731e9b0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.92273064Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.92273539Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.92273892Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.92274217Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:211"}
{"msg":"No ledger in /pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922779791Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:189"}
{"msg":"No ledger in /pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922793061Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:189"}
{"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922801871Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"}
{"msg":"No ledger in /pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922811441Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:189"}
{"msg":"No ledger in /pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922818511Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:189"}
{"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922822651Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"}
{"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922852672Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:216"}
{"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.922863142Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:231"}
{"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.933662212Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:743"}
{"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.933700153Z","hostname":"sock","pid":1566,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999917039s","next_cleanup":"Instant { tv_sec: 1388, tv_nsec: 805540073 }"}
{"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.933755344Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4005"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.933766224Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"No ledger in /pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.933814105Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:189"}
{"msg":"No ledger in /pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.936180742Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:189"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.936532078Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"}
{"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.938206424Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:51","local_addr":"[fdb0:200::1]:12346"}
{"msg":"Injecting RSS configuration: RackInitializeRequest {\n    trust_quorum_peers: None,\n    bootstrap_discovery: OnlyOurs,\n    ntp_servers: [\n        \"0.pool.ntp.org\",\n    ],\n    dns_servers: [\n        1.1.1.1,\n        9.9.9.9,\n    ],\n    internal_services_ip_pool_ranges: [\n        V4(\n            Ipv4Range {\n                first: 10.151.2.164,\n                last: 10.151.2.173,\n            },\n        ),\n    ],\n    external_dns_ips: [\n        10.151.2.164,\n        10.151.2.165,\n    ],\n    external_dns_zone_name: \"oxide.test\",\n    external_certificates: \"<redacted>\",\n    recovery_silo: RecoverySiloConfig {\n        silo_name: Name(\n            \"recovery\",\n        ),\n        user_name: UserId(\n            \"recovery\",\n        ),\n        user_password_hash: NewPasswordHash(\n            PasswordHashString {\n                string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n                encoding: B64,\n            },\n        ),\n    },\n    rack_network_config: RackNetworkConfigV2 {\n        rack_subnet: Ipv6Net {\n            addr: fd00:1122:3344:100::,\n            width: 56,\n        },\n        infra_ip_first: 10.151.2.174,\n        infra_ip_last: 10.151.2.174,\n        ports: [\n            PortConfigV2 {\n                routes: [\n                    RouteConfig {\n                        destination: V4(\n                            Ipv4Net {\n                                addr: 0.0.0.0,\n                                width: 0,\n                            },\n                        ),\n                        nexthop: 10.151.2.100,\n                        vlan_id: None,\n                        rib_priority: None,\n                    },\n                ],\n                addresses: [\n                    UplinkAddressConfig {\n                        address: V4(\n                            Ipv4Net {\n                                addr: 10.151.2.174,\n                                width: 24,\n                            },\n                        ),\n                        vlan_id: None,\n                    },\n                ],\n                switch: Switch0,\n                port: \"qsfp0\",\n                uplink_port_speed: Speed40G,\n                uplink_port_fec: Some(\n                    None,\n                ),\n                bgp_peers: [],\n                autoneg: false,\n                lldp: None,\n                tx_eq: None,\n            },\n        ],\n        bgp: [],\n        bfd: [],\n    },\n    allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.938787744Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1200"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.938794234Z","hostname":"sock","pid":1566,"component":"DnsResolver","component":"RSS","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.938846854Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"No ledger in /pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.938894655Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:189"}
{"msg":"No ledger in /pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.938925816Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:189"}
{"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.938930316Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1263"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.938952206Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"No ledger in /pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.938962276Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:189"}
{"msg":"No ledger in /pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.938970087Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:189"}
{"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.938977437Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1305"}
{"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.938980907Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:156"}
{"msg":"Creating plan for the sled at fdb0:200::1","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.938984357Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:129"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.939014897Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Writing ledger to /pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.939038848Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:200"}
{"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.939347882Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:133","path":"\"/pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/rss-sled-plan.json\""}
{"msg":"Writing ledger to /pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.939353653Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:200"}
{"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.939495695Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:133","path":"\"/pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/rss-sled-plan.json\""}
{"msg":"Sled plan written to storage: Plan {\n    rack_id: c1b8264e-5605-46dd-a3d8-c05674dfacdb,\n    sleds: {\n        [fdb0:200::1]:12346: StartSledAgentRequest {\n            generation: 0,\n            schema_version: 1,\n            body: StartSledAgentRequestBody {\n                id: df02b17d-a4bd-4ac6-9447-5abe2628a534 (sled),\n                rack_id: c1b8264e-5605-46dd-a3d8-c05674dfacdb,\n                use_trust_quorum: false,\n                is_lrtq_learner: false,\n                subnet: Ipv6Subnet {\n                    net: Ipv6Net {\n                        addr: fd00:1122:3344:101::,\n                        width: 64,\n                    },\n                },\n            },\n        },\n    },\n    config: RackInitializeRequest {\n        trust_quorum_peers: None,\n        bootstrap_discovery: OnlyOurs,\n        ntp_servers: [\n            \"0.pool.ntp.org\",\n        ],\n        dns_servers: [\n            1.1.1.1,\n            9.9.9.9,\n        ],\n        internal_services_ip_pool_ranges: [\n            V4(\n                Ipv4Range {\n                    first: 10.151.2.164,\n                    last: 10.151.2.173,\n                },\n            ),\n        ],\n        external_dns_ips: [\n            10.151.2.164,\n            10.151.2.165,\n        ],\n        external_dns_zone_name: \"oxide.test\",\n        external_certificates: \"<redacted>\",\n        recovery_silo: RecoverySiloConfig {\n            silo_name: Name(\n                \"recovery\",\n            ),\n            user_name: UserId(\n                \"recovery\",\n            ),\n            user_password_hash: NewPasswordHash(\n                PasswordHashString {\n                    string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n                    encoding: B64,\n                },\n            ),\n        },\n        rack_network_config: RackNetworkConfigV2 {\n            rack_subnet: Ipv6Net {\n                addr: fd00:1122:3344:100::,\n                width: 56,\n            },\n            infra_ip_first: 10.151.2.174,\n            infra_ip_last: 10.151.2.174,\n            ports: [\n                PortConfigV2 {\n                    routes: [\n                        RouteConfig {\n                            destination: V4(\n                                Ipv4Net {\n                                    addr: 0.0.0.0,\n                                    width: 0,\n                                },\n                            ),\n                            nexthop: 10.151.2.100,\n                            vlan_id: None,\n                            rib_priority: None,\n                        },\n                    ],\n                    addresses: [\n                        UplinkAddressConfig {\n                            address: V4(\n                                Ipv4Net {\n                                    addr: 10.151.2.174,\n                                    width: 24,\n                                },\n                            ),\n                            vlan_id: None,\n                        },\n                    ],\n                    switch: Switch0,\n                    port: \"qsfp0\",\n                    uplink_port_speed: Speed40G,\n                    uplink_port_fec: Some(\n                        None,\n                    ),\n                    bgp_peers: [],\n                    autoneg: false,\n                    lldp: None,\n                    tx_eq: None,\n                },\n            ],\n            bgp: [],\n            bfd: [],\n        },\n        allowed_source_ips: Any,\n    },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.939592756Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:176"}
{"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.939598446Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1339"}
{"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.939631347Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"}
{"msg":"Writing ledger to /pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.939644447Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:200"}
{"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.939795209Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:133","path":"\"/pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/cluster/bootstore-network-config.json\""}
{"msg":"Writing ledger to /pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.93980136Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:200"}
{"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.939923591Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:133","path":"\"/pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/cluster/bootstore-network-config.json\""}
{"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.939929202Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"}
{"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.939964642Z","hostname":"sock","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:200::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: df02b17d-a4bd-4ac6-9447-5abe2628a534 (sled), rack_id: c1b8264e-5605-46dd-a3d8-c05674dfacdb, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"}
{"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.940558122Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4302"}
{"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.940620683Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4702"}
{"msg":"Loaded keys and certs","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.94487931Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:200::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/client.rs:50"}
{"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.950123723Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:200::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"}
{"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.950320306Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:200::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"}
{"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.954937189Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"}
{"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.95499733Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"}
{"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.955113112Z","hostname":"sock","pid":1566,"remote_addr":"[fdb0:200::1]:46004","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:74"}
{"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: df02b17d-a4bd-4ac6-9447-5abe2628a534 (sled), rack_id: c1b8264e-5605-46dd-a3d8-c05674dfacdb, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.955257204Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:362"}
{"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.955273994Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:377"}
{"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.955302775Z","hostname":"sock","pid":1566,"file":"sled-agent/src/server.rs:44"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.955309815Z","hostname":"sock","pid":1566,"component":"DnsResolver","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"}
{"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.955317365Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.955331765Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:809"}
{"msg":"No ledger in /pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.955343765Z","hostname":"sock","pid":1566,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:189"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.955359656Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"}
{"msg":"No ledger in /pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.955365686Z","hostname":"sock","pid":1566,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:189"}
{"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.955375946Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:798"}
{"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.955381896Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:826"}
{"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.955387886Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:592"}
{"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.955409676Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"}
{"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.955420836Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:697","result":"DisksManagementResult { status: [] }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.971748404Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:00.992823947Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"}
{"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.00503269Z","hostname":"sock","pid":1566,"sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:407"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.005093901Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.005101881Z","hostname":"sock","pid":1566,"sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:419"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.005110072Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 127.547037ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.007808734Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f5879111-af20-4bcd-869c-07248f29867b\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 23:10:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f5879111-af20-4bcd-869c-07248f29867b\" }"}
{"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.010457586Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"}
{"msg":"attempting to create encrypted zvol: name=\"oxi_67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.018134757Z","hostname":"sock","pid":1566,"file":"sled-agent/src/swap_device.rs:153"}
{"msg":"successfully created encrypted zvol: name=\"oxi_67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.041536607Z","hostname":"sock","pid":1566,"file":"sled-agent/src/swap_device.rs:249"}
{"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.041579088Z","hostname":"sock","pid":1566,"file":"sled-agent/src/swap_device.rs:96"}
{"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.046352233Z","hostname":"sock","pid":1566,"sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:434"}
{"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.046365384Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:128"}
{"msg":"Ensuring dataset oxi_67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.046370234Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:138"}
{"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.13591428Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:173"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.13596491Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 96.225846ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.138550601Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"28cc0327-dfbf-4b9b-aa7f-e7384680e538\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 23:10:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"28cc0327-dfbf-4b9b-aa7f-e7384680e538\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.235642766Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 93.41407ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.236187985Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c7e1b5e3-ed42-4c4d-974d-a3c716d63cfc\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 23:10:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c7e1b5e3-ed42-4c4d-974d-a3c716d63cfc\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.331265038Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 178.906671ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.331751956Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b5db006a-b1bc-4681-ac59-1d5580b90426\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 23:10:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b5db006a-b1bc-4681-ac59-1d5580b90426\" }"}
{"msg":"Mounting oxi_67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.354517685Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:181"}
{"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.367552232Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"}
{"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.367623233Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"}
{"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.367640833Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"}
{"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.367670984Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:195"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.511785532Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 287.644536ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.512543394Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d773474f-ba71-47cc-a72a-097885da1109\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 23:10:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d773474f-ba71-47cc-a72a-097885da1109\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.801278929Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 260.001155ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:01.802152233Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0b57f4de-33de-490a-8d4a-9c2fedcdd1fd\", \"content-length\": \"124\", \"date\": \"Thu, 21 Nov 2024 23:10:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0b57f4de-33de-490a-8d4a-9c2fedcdd1fd\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:02.063162688Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:02.095915906Z","hostname":"sock","pid":1566,"sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:93"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:02.101434923Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:64902","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327","component":"producer-server","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:02.101469144Z","hostname":"sock","pid":1566,"component":"internal-dns-resolver","file":"internal-dns/resolver/src/resolver.rs:111","component":"producer-server","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"}
{"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:02.101523935Z","hostname":"sock","pid":1566,"file":"oximeter/producer/src/lib.rs:279","component":"producer-server","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/producer/src/lib.rs:279","interval":"30s","address":"[fd00:1122:3344:101::1]:64902","producer_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:02.102600052Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"204.531873ms"}
{"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:02.103674649Z","hostname":"sock","pid":1566,"component":"VmmReservoirManager","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"}
{"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:02.103681649Z","hostname":"sock","pid":1566,"component":"VmmReservoirManager","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"}
{"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:02.10371858Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:927","underlay_address":"fd00:1122:3344:101::1"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:02.10373659Z","hostname":"sock","pid":1566,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:02.308261713Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"641.492307ms"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:02.951273269Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"790.096596ms"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:03.742948255Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.996907644s"}
{"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:04.71733083Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2676"}
{"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:04.71736667Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2921"}
{"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:04.7173722Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2546"}
{"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:04.71737676Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2593"}
{"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:04.71738823Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3002"}
{"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:04.717392701Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3047"}
{"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:04.719368392Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2846"}
{"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:04.721017588Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2587"}
{"msg":"Profile for oxz_switch:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService0'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='::1'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/mgs\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"address\" value='[::1]:12225'/>\n        <propval type=\"astring\" name=\"id\" value='d7a26d40-543f-47f0-8255-7216f60ee965'/>\n        <propval type=\"astring\" name=\"rack_id\" value='c1b8264e-5605-46dd-a3d8-c05674dfacdb'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/wicketd\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"address\" value='[::1]:12226'/>\n        <propval type=\"astring\" name=\"artifact-address\" value='[fdb0:200::2]:12227'/>\n        <propval type=\"astring\" name=\"baseboard-file\" value='/opt/oxide/baseboard.json'/>\n        <propval type=\"astring\" name=\"mgs-address\" value='[::1]:12225'/>\n        <propval type=\"astring\" name=\"nexus-proxy-address\" value='[::]:12229'/>\n        <propval type=\"astring\" name=\"rack-subnet\" value='fd00:1122:3344::'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/switch_zone_setup\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"gz_local_link_addr\" value='fe80::8:20ff:fe35:17d4'/>\n        <property type=\"astring\" name=\"link_local_links\">\n          <astring_list>\n            <value_node value=\"tfportrear0_0\" />\n            <value_node value=\"oxBootstrap0\" />\n          </astring_list>\n        </property>\n        <propval type=\"astring\" name=\"bootstrap_addr\" value='fdb0:200::2'/>\n        <propval type=\"astring\" name=\"bootstrap_vnic\" value='oxBootstrap0'/>\n        <propval type=\"astring\" name=\"baseboard_info\" value='{\"type\":\"pc\",\"identifier\":\"sock\",\"model\":\"i86pc\"}'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/dendrite\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"rack_id\" value='c1b8264e-5605-46dd-a3d8-c05674dfacdb'/>\n        <propval type=\"astring\" name=\"sled_id\" value='df02b17d-a4bd-4ac6-9447-5abe2628a534'/>\n        <propval type=\"astring\" name=\"sled_model\" value='i86pc'/>\n        <propval type=\"astring\" name=\"sled_serial\" value='sock'/>\n        <propval type=\"astring\" name=\"sled_revision\" value='0'/>\n        <propval type=\"astring\" name=\"address\" value='[::1]:12224'/>\n        <propval type=\"astring\" name=\"front_ports\" value='1'/>\n        <propval type=\"astring\" name=\"rear_ports\" value='1'/>\n        <propval type=\"astring\" name=\"port_config\" value='/opt/oxide/dendrite/misc/softnpu_single_sled_config.toml'/>\n        <propval type=\"astring\" name=\"mgmt\" value='uds'/>\n        <propval type=\"astring\" name=\"uds_path\" value='/opt/softnpu/stuff'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/tfport\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"host\" value='[::1]'/>\n        <propval type=\"astring\" name=\"port\" value='12224'/>\n        <propval type=\"astring\" name=\"flags\" value='--sync-only'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/lldpd\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"board_rev\" value='softnpu_front_1_rear_1'/>\n        <propval type=\"astring\" name=\"scrimlet_id\" value='sock'/>\n        <propval type=\"astring\" name=\"scrimlet_model\" value='i86pc'/>\n        <propval type=\"astring\" name=\"address\" value='[::1]:12230'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/pumpkind\">\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/mgd\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"sled_uuid\" value='df02b17d-a4bd-4ac6-9447-5abe2628a534'/>\n        <propval type=\"astring\" name=\"rack_uuid\" value='c1b8264e-5605-46dd-a3d8-c05674dfacdb'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/mg-ddm\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"mode\" value='transit'/>\n        <propval type=\"astring\" name=\"dendrite\" value='true'/>\n        <propval type=\"astring\" name=\"sled_uuid\" value='df02b17d-a4bd-4ac6-9447-5abe2628a534'/>\n        <propval type=\"astring\" name=\"rack_uuid\" value='c1b8264e-5605-46dd-a3d8-c05674dfacdb'/>\n        <propval type=\"astring\" name=\"interfaces\" value='tfportrear0_0/ll'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/uplink\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:04.72114723Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:04.721261482Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 62.191971ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:05.038313814Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 75.213576ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:05.270044308Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 292.00796ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:05.483569134Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 501.172302ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:05.876826121Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 985.813726ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:06.545944369Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:06.742064399Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"3.667359106s"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.451088322s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:07.742707799Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.20169408s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:09.370121777Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:10.411511141Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"10.865636904s"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:10.972051922Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:10.97255011Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12348","component":"dropshot (Repo Depot)","component":"ArtifactStore","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:10.973692748Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"}
{"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:10.973707508Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:432"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:10.973725228Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Writing ledger to /pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:10.973749249Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"common/src/ledger.rs:200"}
{"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:10.973909331Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"common/src/ledger.rs:133","path":"\"/pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/sled-agent-request.json\""}
{"msg":"Writing ledger to /pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:10.973914921Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"common/src/ledger.rs:200"}
{"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:10.974043333Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"common/src/ledger.rs:133","path":"\"/pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/sled-agent-request.json\""}
{"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:10.974100194Z","hostname":"sock","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"}
{"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:10.974154845Z","hostname":"sock","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.2.100, vlan_id: None, rib_priority: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: Some(None), bgp_peers: [], autoneg: false, lldp: None, tx_eq: None }], bgp: [], bfd: [] })))"}
{"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:10.974185825Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4005"}
{"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:10.974201576Z","hostname":"sock","pid":1566,"remote_addr":"[fdb0:200::1]:46004","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:81"}
{"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.161602629Z","hostname":"sock","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:117","peer":"[fdb0:200::1]:12346"}
{"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.16171498Z","hostname":"sock","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:252","target_sled":"[fdb0:200::1]:12346"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.161824102Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"No ledger in /pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/rss-service-plan-v5.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.161900543Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:189"}
{"msg":"No ledger in /pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/rss-service-plan-v5.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.161907304Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:189"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.161921984Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.161949134Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.161969744Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.161994675Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.167465881Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4326","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"}
{"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.167484782Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4342"}
{"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.167491802Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:534"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.427615954Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:57689"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.428008751Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.428016131Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"No ledger in /pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.428070481Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:189"}
{"msg":"No ledger in /pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.428075472Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:189"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.428141353Z","hostname":"sock","pid":1566,"uri":"/inventory","method":"GET","req_id":"56c8253c-d61c-4d49-aec2-453396384407","remote_addr":"[fd00:1122:3344:101::1]:57689","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":269,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.460056657Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:61984"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.460185239Z","hostname":"sock","pid":1566,"uri":"/sled-role","method":"GET","req_id":"2320653c-6c3a-42d5-a55b-b9d469809d89","remote_addr":"[fd00:1122:3344:101::1]:61984","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":40,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.461686793Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Writing ledger to /pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/.rss-service-plan-v5.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.461727004Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:200"}
{"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.46210801Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:133","path":"\"/pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/rss-service-plan-v5.json\""}
{"msg":"Writing ledger to /pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/.rss-service-plan-v5.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.46211488Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:200"}
{"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.462351963Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:133","path":"\"/pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/rss-service-plan-v5.json\""}
{"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.462358443Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:980"}
{"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: da5d01e2-22ea-4989-a7da-b44302e9ed18, pool_id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 60180f3b-296a-49fb-a3eb-db0529d85e18, pool_id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 28290b8e-4fb6-4153-b460-834e1a98721e, pool_id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 3288a0fb-dad8-4021-860d-9429ebb6197d, pool_id: 60d88883-2ddc-48f7-92be-fb3130552398 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: e857f345-58fb-438d-a07d-a43becfcfd7d, pool_id: 56581dc2-c967-4930-aa2d-7f607fddd15e (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 93d3ee9e-4d54-488f-a3e6-4699a4c8a5b7, pool_id: 4a60e634-3d93-4efd-bc48-2e2658b8d062 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 21f7ea66-295c-4532-85e0-835391a4fd27, pool_id: 2735b386-4b94-446b-b6e4-69280ab9e40a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: ba313001-226e-4c7c-964d-f5b4e3b4a86b, pool_id: e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 67b8c359-7627-4b69-aaf1-d558c2108883, pool_id: d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1 (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.493687709Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:382"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.49375621Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:42314"}
{"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.494009024Z","hostname":"sock","pid":1566,"sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:884"}
{"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: da5d01e2-22ea-4989-a7da-b44302e9ed18, pool_id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 60180f3b-296a-49fb-a3eb-db0529d85e18, pool_id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 28290b8e-4fb6-4153-b460-834e1a98721e, pool_id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 3288a0fb-dad8-4021-860d-9429ebb6197d, pool_id: 60d88883-2ddc-48f7-92be-fb3130552398 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: e857f345-58fb-438d-a07d-a43becfcfd7d, pool_id: 56581dc2-c967-4930-aa2d-7f607fddd15e (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 93d3ee9e-4d54-488f-a3e6-4699a4c8a5b7, pool_id: 4a60e634-3d93-4efd-bc48-2e2658b8d062 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 21f7ea66-295c-4532-85e0-835391a4fd27, pool_id: 2735b386-4b94-446b-b6e4-69280ab9e40a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: ba313001-226e-4c7c-964d-f5b4e3b4a86b, pool_id: e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 67b8c359-7627-4b69-aaf1-d558c2108883, pool_id: d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.494062175Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"No ledger in /pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.494067215Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:189"}
{"msg":"No ledger in /pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.494073065Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:189"}
{"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.494078065Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1175"}
{"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.494092735Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.494097625Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.494103385Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.494108096Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.621116733Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:11.621155744Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool)"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/df02b17d-a4bd-4ac6-9447-5abe2628a534)","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:12.074595703Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"}
{"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.296489311Z","hostname":"sock","pid":1566,"component":"VmmReservoirManager","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:12.297932314Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:12.298067476Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.344460209Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.352436315Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.352516947Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.352526137Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.352544827Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.352561457Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.35268909Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.35269782Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.388836951Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.396821287Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"}
{"msg":"Looking for unencrypted datasets in oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.57473442Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.58294257Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.58298119Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.583005141Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.583012581Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.583028511Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.592905167Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.592925537Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.678778495Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.686418406Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.686448476Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.686470876Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.686476937Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.686486557Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.686617189Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.686624409Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.723437161Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.730905769Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"}
{"msg":"Looking for unencrypted datasets in oxp_bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.8998435Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.906608177Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.906654197Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.906663648Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.906672848Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.906689568Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.916205239Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:12.916236669Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.013464685Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.021877138Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.02194143Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.02195309Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.02199892Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.022014341Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.022146893Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.022155253Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.057225857Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.065679301Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/df02b17d-a4bd-4ac6-9447-5abe2628a534)","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:13.077520278Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"}
{"msg":"Looking for unencrypted datasets in oxp_e858b5e9-aa1b-415e-9d0d-1b75991b7849","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.240671028Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.248269558Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.248308158Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.248322209Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.248329309Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.248338059Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.257745318Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.257765248Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"60d88883-2ddc-48f7-92be-fb3130552398 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.348696785Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 60d88883-2ddc-48f7-92be-fb3130552398 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.355549864Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.355576024Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.355582314Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.355602935Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.355621265Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.355746257Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.355753097Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.388651077Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_60d88883-2ddc-48f7-92be-fb3130552398/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.395708569Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"}
{"msg":"Looking for unencrypted datasets in oxp_60d88883-2ddc-48f7-92be-fb3130552398","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.553010666Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.559742592Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.559779443Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.559786853Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.559795063Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.559799603Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.568543991Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.568563472Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"56581dc2-c967-4930-aa2d-7f607fddd15e (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.655234492Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 56581dc2-c967-4930-aa2d-7f607fddd15e (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.662262913Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.662287743Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.662293754Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.662326524Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.662382415Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.662550458Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.662561488Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.697422029Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_56581dc2-c967-4930-aa2d-7f607fddd15e/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.705370384Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"}
{"msg":"Looking for unencrypted datasets in oxp_56581dc2-c967-4930-aa2d-7f607fddd15e","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.863040827Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.869859235Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.869898326Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.869918426Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.869935626Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.869957146Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.879489797Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.879527088Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"4a60e634-3d93-4efd-bc48-2e2658b8d062 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.968755328Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: 4a60e634-3d93-4efd-bc48-2e2658b8d062 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.97644568Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.976507471Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.976520731Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.976586712Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.976595752Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.976760465Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:13.976768825Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.010691702Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_4a60e634-3d93-4efd-bc48-2e2658b8d062/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.018613227Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/df02b17d-a4bd-4ac6-9447-5abe2628a534)","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:14.080071478Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"}
{"msg":"Looking for unencrypted datasets in oxp_4a60e634-3d93-4efd-bc48-2e2658b8d062","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.196130463Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.204197121Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.204236311Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.204243931Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.204250862Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.204255292Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.214556254Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.214585125Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"2735b386-4b94-446b-b6e4-69280ab9e40a (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.308986678Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 2735b386-4b94-446b-b6e4-69280ab9e40a (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.317000864Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.317037875Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.317044055Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.317056415Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.317065735Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.317190277Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.317197017Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.353750735Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_2735b386-4b94-446b-b6e4-69280ab9e40a/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.361556509Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"}
{"msg":"Looking for unencrypted datasets in oxp_2735b386-4b94-446b-b6e4-69280ab9e40a","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.538068359Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.54632405Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.54636056Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.546375281Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.546380911Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.546386681Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.556126855Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.556153565Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.651126927Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.659602451Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.659628551Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.659634691Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.659670752Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.659683392Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.659805784Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.659812324Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.693851882Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.702414707Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"}
{"msg":"Looking for unencrypted datasets in oxp_e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.89167206Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.899841619Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.899885249Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.89989287Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.89990229Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.89991703Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.91067522Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:14.9106949Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.009033405Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.017931135Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.017967136Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.017974736Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.018011987Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.018041977Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.018150839Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.018159149Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.054716017Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.06312217Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/df02b17d-a4bd-4ac6-9447-5abe2628a534)","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:15.081875906Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"}
{"msg":"Looking for unencrypted datasets in oxp_d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.242007818Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.249995824Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.250034795Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Writing ledger to /pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.250102226Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:200"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.250228998Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 67a39b8e-1dc3-40e2-8b0f-59c429bf4f91 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4c94aa86-6837-420b-a3c1-83899731e9b0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 60d88883-2ddc-48f7-92be-fb3130552398 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 56581dc2-c967-4930-aa2d-7f607fddd15e (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4a60e634-3d93-4efd-bc48-2e2658b8d062 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 2735b386-4b94-446b-b6e4-69280ab9e40a (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1 (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.250295109Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.250303199Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.250405021Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/omicron-physical-disks.json\""}
{"msg":"Writing ledger to /pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.250413721Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:200"}
{"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.250620274Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/omicron-physical-disks.json\""}
{"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.250704955Z","hostname":"sock","pid":1566,"sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:888"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.250720575Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.250725746Z","hostname":"sock","pid":1566,"sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:914","generation":"Generation(1)"}
{"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.250735436Z","hostname":"sock","pid":1566,"sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:919"}
{"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.250741966Z","hostname":"sock","pid":1566,"sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:924"}
{"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.250782676Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:122","gen":"Generation(1)"}
{"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.250791297Z","hostname":"sock","pid":1566,"sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:930"}
{"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.250835997Z","hostname":"sock","pid":1566,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:799","gen":"Generation(1)"}
{"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.250845448Z","hostname":"sock","pid":1566,"sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:935"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.250945189Z","hostname":"sock","pid":1566,"uri":"/omicron-physical-disks","method":"PUT","req_id":"52a5611b-b77f-403a-99e0-74f3d6bf2e76","remote_addr":"[fd00:1122:3344:101::1]:42314","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3757077,"response_code":"200"}
{"msg":"attempting to set up sled's datasets: DatasetsConfig { generation: Generation(1), datasets: {0646288e-6664-4bdf-a5b8-0cbecb247d11 (dataset): DatasetConfig { id: 0646288e-6664-4bdf-a5b8-0cbecb247d11 (dataset), name: DatasetName { pool_name: ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_4fb64aa2-0fe1-47d4-b447-fc7e161481c6\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0686ebfa-21bb-4109-811d-839920f74823 (dataset): DatasetConfig { id: 0686ebfa-21bb-4109-811d-839920f74823 (dataset), name: DatasetName { pool_name: ZpoolName { id: 56581dc2-c967-4930-aa2d-7f607fddd15e (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1325c7a2-b88c-42d0-a166-3debdf30f7a2 (dataset): DatasetConfig { id: 1325c7a2-b88c-42d0-a166-3debdf30f7a2 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 17794c24-0720-4e81-80e3-60d82f305f0e (dataset): DatasetConfig { id: 17794c24-0720-4e81-80e3-60d82f305f0e (dataset), name: DatasetName { pool_name: ZpoolName { id: 56581dc2-c967-4930-aa2d-7f607fddd15e (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_23f9a8fe-7b99-4a86-9e17-b09185836552\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1a17d0d5-7757-4b23-ab0d-80f680bf0068 (dataset): DatasetConfig { id: 1a17d0d5-7757-4b23-ab0d-80f680bf0068 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4a60e634-3d93-4efd-bc48-2e2658b8d062 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 20783037-22f1-416c-8917-dac8b14521eb (dataset): DatasetConfig { id: 20783037-22f1-416c-8917-dac8b14521eb (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2811d626-f66e-4d13-bd5a-3a01ddfbfde4 (dataset): DatasetConfig { id: 2811d626-f66e-4d13-bd5a-3a01ddfbfde4 (dataset), name: DatasetName { pool_name: ZpoolName { id: 56581dc2-c967-4930-aa2d-7f607fddd15e (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 28d69dfb-2f52-4937-b3c8-a6724fb9a8fd (dataset): DatasetConfig { id: 28d69dfb-2f52-4937-b3c8-a6724fb9a8fd (dataset), name: DatasetName { pool_name: ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2ab271d4-0db3-45b0-bae0-c99dd5d49fda (dataset): DatasetConfig { id: 2ab271d4-0db3-45b0-bae0-c99dd5d49fda (dataset), name: DatasetName { pool_name: ZpoolName { id: e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2c6bb6c1-92bb-4dd8-9cd2-dc78d8b82cd8 (dataset): DatasetConfig { id: 2c6bb6c1-92bb-4dd8-9cd2-dc78d8b82cd8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_4a49f2d4-4a63-438e-9662-856d67052865\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 31d5e35b-4e3d-4dc9-a651-aa95fd6d7ae3 (dataset): DatasetConfig { id: 31d5e35b-4e3d-4dc9-a651-aa95fd6d7ae3 (dataset), name: DatasetName { pool_name: ZpoolName { id: 2735b386-4b94-446b-b6e4-69280ab9e40a (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 33570daf-d204-487c-a3dc-d340d67bba0b (dataset): DatasetConfig { id: 33570daf-d204-487c-a3dc-d340d67bba0b (dataset), name: DatasetName { pool_name: ZpoolName { id: 2735b386-4b94-446b-b6e4-69280ab9e40a (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_4a05a39f-e9d2-455e-9cb5-f73902d6694b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 353f3312-9495-4467-81ee-23cd15a885a4 (dataset): DatasetConfig { id: 353f3312-9495-4467-81ee-23cd15a885a4 (dataset), name: DatasetName { pool_name: ZpoolName { id: d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 359e87c3-8099-4007-9e6f-8d41294a1131 (dataset): DatasetConfig { id: 359e87c3-8099-4007-9e6f-8d41294a1131 (dataset), name: DatasetName { pool_name: ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 36a1200b-6191-4278-a627-29c085136adc (dataset): DatasetConfig { id: 36a1200b-6191-4278-a627-29c085136adc (dataset), name: DatasetName { pool_name: ZpoolName { id: 60d88883-2ddc-48f7-92be-fb3130552398 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3a07ae54-af6a-4508-a3c7-3803a37a1596 (dataset): DatasetConfig { id: 3a07ae54-af6a-4508-a3c7-3803a37a1596 (dataset), name: DatasetName { pool_name: ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 41174fca-d60b-4637-bde3-6a806ae28192 (dataset): DatasetConfig { id: 41174fca-d60b-4637-bde3-6a806ae28192 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 43e11876-b574-4e96-97e4-46898d4fb2d6 (dataset): DatasetConfig { id: 43e11876-b574-4e96-97e4-46898d4fb2d6 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4477a59d-7801-420d-8c12-be7ef25d4730 (dataset): DatasetConfig { id: 4477a59d-7801-420d-8c12-be7ef25d4730 (dataset), name: DatasetName { pool_name: ZpoolName { id: 56581dc2-c967-4930-aa2d-7f607fddd15e (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_1a248290-007e-4b53-b1f1-6f7a9e3dbb71\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4844bd22-3782-406a-8d61-8603fefe283f (dataset): DatasetConfig { id: 4844bd22-3782-406a-8d61-8603fefe283f (dataset), name: DatasetName { pool_name: ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_9e92e318-bc88-437a-bf0e-cb6cc571e7b1\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 49482455-cdbe-4899-94bc-9d4244474e82 (dataset): DatasetConfig { id: 49482455-cdbe-4899-94bc-9d4244474e82 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_6ca40c88-6f5e-4727-8259-0993d419cff1\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4a53bade-703d-4a8e-b8ba-693884077454 (dataset): DatasetConfig { id: 4a53bade-703d-4a8e-b8ba-693884077454 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4ab918c0-5ef8-4404-9ede-17b689514f13 (dataset): DatasetConfig { id: 4ab918c0-5ef8-4404-9ede-17b689514f13 (dataset), name: DatasetName { pool_name: ZpoolName { id: 60d88883-2ddc-48f7-92be-fb3130552398 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 4b04a7a7-55a3-457d-ae8b-4302aaa428ab (dataset): DatasetConfig { id: 4b04a7a7-55a3-457d-ae8b-4302aaa428ab (dataset), name: DatasetName { pool_name: ZpoolName { id: 60d88883-2ddc-48f7-92be-fb3130552398 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_710d532b-120e-4fdc-9cae-a0252c955c30\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5cf204c8-c305-4d6f-89b2-9415ddc411e3 (dataset): DatasetConfig { id: 5cf204c8-c305-4d6f-89b2-9415ddc411e3 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_5267fc4d-d51f-4cd0-9820-0c103fa42eea\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6420ee84-a440-4ee2-9ee7-64cd807d04bd (dataset): DatasetConfig { id: 6420ee84-a440-4ee2-9ee7-64cd807d04bd (dataset), name: DatasetName { pool_name: ZpoolName { id: 60d88883-2ddc-48f7-92be-fb3130552398 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_c5e898ed-4202-4777-9b7c-9d08a9bd3c3a\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 64936e38-f87d-48f7-9fe9-a73f614a1c98 (dataset): DatasetConfig { id: 64936e38-f87d-48f7-9fe9-a73f614a1c98 (dataset), name: DatasetName { pool_name: ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6b4cad43-4f31-4ee6-baf3-34133fb352f9 (dataset): DatasetConfig { id: 6b4cad43-4f31-4ee6-baf3-34133fb352f9 (dataset), name: DatasetName { pool_name: ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }, kind: TransientZone { name: \"oxz_oximeter_13edfd02-a016-4abd-b9a3-b2af39dc85f9\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6e082723-189a-4ae8-ad1d-9c788e5ad902 (dataset): DatasetConfig { id: 6e082723-189a-4ae8-ad1d-9c788e5ad902 (dataset), name: DatasetName { pool_name: ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 725d02e7-ef0b-45b6-9daf-400d782153b0 (dataset): DatasetConfig { id: 725d02e7-ef0b-45b6-9daf-400d782153b0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 2735b386-4b94-446b-b6e4-69280ab9e40a (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 7eac0f17-2a31-4c63-84f2-30ad51a8c1bc (dataset): DatasetConfig { id: 7eac0f17-2a31-4c63-84f2-30ad51a8c1bc (dataset), name: DatasetName { pool_name: ZpoolName { id: d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7f63b141-11a6-4d53-8426-fe2d8c92530b (dataset): DatasetConfig { id: 7f63b141-11a6-4d53-8426-fe2d8c92530b (dataset), name: DatasetName { pool_name: ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 85304f96-cc43-4611-9315-f1ba7ead5e8b (dataset): DatasetConfig { id: 85304f96-cc43-4611-9315-f1ba7ead5e8b (dataset), name: DatasetName { pool_name: ZpoolName { id: d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_8796224f-ce1e-4f10-8b1b-63b2661798e0\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 86128a35-b205-4776-b2f1-2fa5df01c4b9 (dataset): DatasetConfig { id: 86128a35-b205-4776-b2f1-2fa5df01c4b9 (dataset), name: DatasetName { pool_name: ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_62b89797-eabf-4a54-ad84-6278e0d63bb2\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 87f7eea6-4e2b-4c9f-b8a7-1676f62d84ab (dataset): DatasetConfig { id: 87f7eea6-4e2b-4c9f-b8a7-1676f62d84ab (dataset), name: DatasetName { pool_name: ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8eb45c40-0d38-4946-828f-b00637a20c18 (dataset): DatasetConfig { id: 8eb45c40-0d38-4946-828f-b00637a20c18 (dataset), name: DatasetName { pool_name: ZpoolName { id: 2735b386-4b94-446b-b6e4-69280ab9e40a (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 96747428-6a4d-440e-90cd-3aa267ede24f (dataset): DatasetConfig { id: 96747428-6a4d-440e-90cd-3aa267ede24f (dataset), name: DatasetName { pool_name: ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 97fb4d68-ec04-4fff-9663-56b5a3dc9fae (dataset): DatasetConfig { id: 97fb4d68-ec04-4fff-9663-56b5a3dc9fae (dataset), name: DatasetName { pool_name: ZpoolName { id: 56581dc2-c967-4930-aa2d-7f607fddd15e (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 983df6bd-7882-42d2-8191-f3afaae6e762 (dataset): DatasetConfig { id: 983df6bd-7882-42d2-8191-f3afaae6e762 (dataset), name: DatasetName { pool_name: ZpoolName { id: 60d88883-2ddc-48f7-92be-fb3130552398 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_5d56a5dd-553d-43d1-ada2-7c96d8406842\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a1ff2dfd-9de8-46ee-b554-8b069dfb6df2 (dataset): DatasetConfig { id: a1ff2dfd-9de8-46ee-b554-8b069dfb6df2 (dataset), name: DatasetName { pool_name: ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a4fd30f1-4698-41b3-b8fc-cbb83226684e (dataset): DatasetConfig { id: a4fd30f1-4698-41b3-b8fc-cbb83226684e (dataset), name: DatasetName { pool_name: ZpoolName { id: 56581dc2-c967-4930-aa2d-7f607fddd15e (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_87ace5e0-9e49-47a5-9e60-8ef09cb36b5c\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a604712a-c938-4f89-b4b3-d4af9c23c9f8 (dataset): DatasetConfig { id: a604712a-c938-4f89-b4b3-d4af9c23c9f8 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_2a081fd5-0462-4328-9db6-071aae0d221b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b28a4670-0930-4cd4-b1e8-7bd1cc120b2b (dataset): DatasetConfig { id: b28a4670-0930-4cd4-b1e8-7bd1cc120b2b (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, b39c7a53-f320-4198-bafa-49bdea95c19a (dataset): DatasetConfig { id: b39c7a53-f320-4198-bafa-49bdea95c19a (dataset), name: DatasetName { pool_name: ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, b45db4e2-3947-420d-96cc-86dbebebf5e3 (dataset): DatasetConfig { id: b45db4e2-3947-420d-96cc-86dbebebf5e3 (dataset), name: DatasetName { pool_name: ZpoolName { id: 60d88883-2ddc-48f7-92be-fb3130552398 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b858f569-1fe5-42ba-a647-e3f0a40a44e9 (dataset): DatasetConfig { id: b858f569-1fe5-42ba-a647-e3f0a40a44e9 (dataset), name: DatasetName { pool_name: ZpoolName { id: e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, bc8bfb55-ea4b-4103-ab23-417c90b9a95b (dataset): DatasetConfig { id: bc8bfb55-ea4b-4103-ab23-417c90b9a95b (dataset), name: DatasetName { pool_name: ZpoolName { id: 4a60e634-3d93-4efd-bc48-2e2658b8d062 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, bda55e8e-d0c0-4309-bd71-c0a4c93a2ac9 (dataset): DatasetConfig { id: bda55e8e-d0c0-4309-bd71-c0a4c93a2ac9 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c01d9a75-c8ea-48b9-85a5-239afff33a22 (dataset): DatasetConfig { id: c01d9a75-c8ea-48b9-85a5-239afff33a22 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_e339542e-0ade-4538-9cba-05a7a02f2352\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c540951f-b03e-4936-a120-1d29ceab341d (dataset): DatasetConfig { id: c540951f-b03e-4936-a120-1d29ceab341d (dataset), name: DatasetName { pool_name: ZpoolName { id: 4a60e634-3d93-4efd-bc48-2e2658b8d062 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, c8467c06-cc64-41bb-a7dd-4bee12301547 (dataset): DatasetConfig { id: c8467c06-cc64-41bb-a7dd-4bee12301547 (dataset), name: DatasetName { pool_name: ZpoolName { id: 2735b386-4b94-446b-b6e4-69280ab9e40a (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_bac4f2f9-1ebe-4488-aa63-037e893be55f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cd3d1292-cc02-4be2-86c8-bed0e140083e (dataset): DatasetConfig { id: cd3d1292-cc02-4be2-86c8-bed0e140083e (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_852e9877-8d05-4fca-8f0b-0d193b0ac883\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cdd73160-8adf-4ae3-8284-d3a43d0bb817 (dataset): DatasetConfig { id: cdd73160-8adf-4ae3-8284-d3a43d0bb817 (dataset), name: DatasetName { pool_name: ZpoolName { id: d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cfda75af-330b-4e05-bb1c-ce25c890426b (dataset): DatasetConfig { id: cfda75af-330b-4e05-bb1c-ce25c890426b (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_c74c8efd-59c4-42a1-a06d-62bee6df73ab\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, dafd3791-da6e-44fc-877f-acce507145d6 (dataset): DatasetConfig { id: dafd3791-da6e-44fc-877f-acce507145d6 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, dc95eb60-7a86-4bec-8b0e-b0cc49c1c233 (dataset): DatasetConfig { id: dc95eb60-7a86-4bec-8b0e-b0cc49c1c233 (dataset), name: DatasetName { pool_name: ZpoolName { id: e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, dd1f66b6-28d5-4344-865a-a2d9989af950 (dataset): DatasetConfig { id: dd1f66b6-28d5-4344-865a-a2d9989af950 (dataset), name: DatasetName { pool_name: ZpoolName { id: e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_6130c690-a32a-4ce5-a932-f89d5df30ab6\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e52fbe4a-10a9-4e7f-8673-73b6bf967338 (dataset): DatasetConfig { id: e52fbe4a-10a9-4e7f-8673-73b6bf967338 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4a60e634-3d93-4efd-bc48-2e2658b8d062 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_54430eae-0e39-408d-a34d-a9a9a575ad61\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ee752ced-3eca-4e7c-943c-78b64aaa6135 (dataset): DatasetConfig { id: ee752ced-3eca-4e7c-943c-78b64aaa6135 (dataset), name: DatasetName { pool_name: ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }, kind: TransientZone { name: \"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ef77125a-5142-4f6a-b05b-9e99a2ec57ca (dataset): DatasetConfig { id: ef77125a-5142-4f6a-b05b-9e99a2ec57ca (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fae30841-e06b-498a-af67-7a5ea9ae27b4 (dataset): DatasetConfig { id: fae30841-e06b-498a-af67-7a5ea9ae27b4 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_clickhouse_ad64b46c-b548-42d7-a128-733f9dbdafda\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fb00fe6c-d12b-4efd-9abd-e4bd4caab4d2 (dataset): DatasetConfig { id: fb00fe6c-d12b-4efd-9abd-e4bd4caab4d2 (dataset), name: DatasetName { pool_name: ZpoolName { id: 56581dc2-c967-4930-aa2d-7f607fddd15e (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_aa128c71-f260-4997-a503-cd00af8f00a9\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fcbf096a-f4db-4011-9059-2d8f549612a2 (dataset): DatasetConfig { id: fcbf096a-f4db-4011-9059-2d8f549612a2 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fe74d525-d726-4e22-a010-cea47f9145d3 (dataset): DatasetConfig { id: fe74d525-d726-4e22-a010-cea47f9145d3 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ff36f9bd-2e05-47da-8076-7fec6f106100 (dataset): DatasetConfig { id: ff36f9bd-2e05-47da-8076-7fec6f106100 (dataset), name: DatasetName { pool_name: ZpoolName { id: 56581dc2-c967-4930-aa2d-7f607fddd15e (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, fff1e4cc-ae24-4f16-b0c7-a3b33393e85a (dataset): DatasetConfig { id: fff1e4cc-ae24-4f16-b0c7-a3b33393e85a (dataset), name: DatasetName { pool_name: ZpoolName { id: 60d88883-2ddc-48f7-92be-fb3130552398 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }} }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.283811529Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:457"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.28392412Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:39695"}
{"msg":"datasets ensure","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.28450486Z","hostname":"sock","pid":1566,"sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:850"}
{"msg":"Received DatasetsEnsure { config: DatasetsConfig { generation: Generation(1), datasets: {0646288e-6664-4bdf-a5b8-0cbecb247d11 (dataset): DatasetConfig { id: 0646288e-6664-4bdf-a5b8-0cbecb247d11 (dataset), name: DatasetName { pool_name: ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_4fb64aa2-0fe1-47d4-b447-fc7e161481c6\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0686ebfa-21bb-4109-811d-839920f74823 (dataset): DatasetConfig { id: 0686ebfa-21bb-4109-811d-839920f74823 (dataset), name: DatasetName { pool_name: ZpoolName { id: 56581dc2-c967-4930-aa2d-7f607fddd15e (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1325c7a2-b88c-42d0-a166-3debdf30f7a2 (dataset): DatasetConfig { id: 1325c7a2-b88c-42d0-a166-3debdf30f7a2 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 17794c24-0720-4e81-80e3-60d82f305f0e (dataset): DatasetConfig { id: 17794c24-0720-4e81-80e3-60d82f305f0e (dataset), name: DatasetName { pool_name: ZpoolName { id: 56581dc2-c967-4930-aa2d-7f607fddd15e (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_23f9a8fe-7b99-4a86-9e17-b09185836552\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1a17d0d5-7757-4b23-ab0d-80f680bf0068 (dataset): DatasetConfig { id: 1a17d0d5-7757-4b23-ab0d-80f680bf0068 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4a60e634-3d93-4efd-bc48-2e2658b8d062 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 20783037-22f1-416c-8917-dac8b14521eb (dataset): DatasetConfig { id: 20783037-22f1-416c-8917-dac8b14521eb (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2811d626-f66e-4d13-bd5a-3a01ddfbfde4 (dataset): DatasetConfig { id: 2811d626-f66e-4d13-bd5a-3a01ddfbfde4 (dataset), name: DatasetName { pool_name: ZpoolName { id: 56581dc2-c967-4930-aa2d-7f607fddd15e (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 28d69dfb-2f52-4937-b3c8-a6724fb9a8fd (dataset): DatasetConfig { id: 28d69dfb-2f52-4937-b3c8-a6724fb9a8fd (dataset), name: DatasetName { pool_name: ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2ab271d4-0db3-45b0-bae0-c99dd5d49fda (dataset): DatasetConfig { id: 2ab271d4-0db3-45b0-bae0-c99dd5d49fda (dataset), name: DatasetName { pool_name: ZpoolName { id: e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2c6bb6c1-92bb-4dd8-9cd2-dc78d8b82cd8 (dataset): DatasetConfig { id: 2c6bb6c1-92bb-4dd8-9cd2-dc78d8b82cd8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_4a49f2d4-4a63-438e-9662-856d67052865\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 31d5e35b-4e3d-4dc9-a651-aa95fd6d7ae3 (dataset): DatasetConfig { id: 31d5e35b-4e3d-4dc9-a651-aa95fd6d7ae3 (dataset), name: DatasetName { pool_name: ZpoolName { id: 2735b386-4b94-446b-b6e4-69280ab9e40a (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 33570daf-d204-487c-a3dc-d340d67bba0b (dataset): DatasetConfig { id: 33570daf-d204-487c-a3dc-d340d67bba0b (dataset), name: DatasetName { pool_name: ZpoolName { id: 2735b386-4b94-446b-b6e4-69280ab9e40a (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_4a05a39f-e9d2-455e-9cb5-f73902d6694b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 353f3312-9495-4467-81ee-23cd15a885a4 (dataset): DatasetConfig { id: 353f3312-9495-4467-81ee-23cd15a885a4 (dataset), name: DatasetName { pool_name: ZpoolName { id: d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 359e87c3-8099-4007-9e6f-8d41294a1131 (dataset): DatasetConfig { id: 359e87c3-8099-4007-9e6f-8d41294a1131 (dataset), name: DatasetName { pool_name: ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 36a1200b-6191-4278-a627-29c085136adc (dataset): DatasetConfig { id: 36a1200b-6191-4278-a627-29c085136adc (dataset), name: DatasetName { pool_name: ZpoolName { id: 60d88883-2ddc-48f7-92be-fb3130552398 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3a07ae54-af6a-4508-a3c7-3803a37a1596 (dataset): DatasetConfig { id: 3a07ae54-af6a-4508-a3c7-3803a37a1596 (dataset), name: DatasetName { pool_name: ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 41174fca-d60b-4637-bde3-6a806ae28192 (dataset): DatasetConfig { id: 41174fca-d60b-4637-bde3-6a806ae28192 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 43e11876-b574-4e96-97e4-46898d4fb2d6 (dataset): DatasetConfig { id: 43e11876-b574-4e96-97e4-46898d4fb2d6 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4477a59d-7801-420d-8c12-be7ef25d4730 (dataset): DatasetConfig { id: 4477a59d-7801-420d-8c12-be7ef25d4730 (dataset), name: DatasetName { pool_name: ZpoolName { id: 56581dc2-c967-4930-aa2d-7f607fddd15e (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_1a248290-007e-4b53-b1f1-6f7a9e3dbb71\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4844bd22-3782-406a-8d61-8603fefe283f (dataset): DatasetConfig { id: 4844bd22-3782-406a-8d61-8603fefe283f (dataset), name: DatasetName { pool_name: ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_9e92e318-bc88-437a-bf0e-cb6cc571e7b1\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 49482455-cdbe-4899-94bc-9d4244474e82 (dataset): DatasetConfig { id: 49482455-cdbe-4899-94bc-9d4244474e82 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_6ca40c88-6f5e-4727-8259-0993d419cff1\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4a53bade-703d-4a8e-b8ba-693884077454 (dataset): DatasetConfig { id: 4a53bade-703d-4a8e-b8ba-693884077454 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4ab918c0-5ef8-4404-9ede-17b689514f13 (dataset): DatasetConfig { id: 4ab918c0-5ef8-4404-9ede-17b689514f13 (dataset), name: DatasetName { pool_name: ZpoolName { id: 60d88883-2ddc-48f7-92be-fb3130552398 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 4b04a7a7-55a3-457d-ae8b-4302aaa428ab (dataset): DatasetConfig { id: 4b04a7a7-55a3-457d-ae8b-4302aaa428ab (dataset), name: DatasetName { pool_name: ZpoolName { id: 60d88883-2ddc-48f7-92be-fb3130552398 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_710d532b-120e-4fdc-9cae-a0252c955c30\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5cf204c8-c305-4d6f-89b2-9415ddc411e3 (dataset): DatasetConfig { id: 5cf204c8-c305-4d6f-89b2-9415ddc411e3 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_5267fc4d-d51f-4cd0-9820-0c103fa42eea\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6420ee84-a440-4ee2-9ee7-64cd807d04bd (dataset): DatasetConfig { id: 6420ee84-a440-4ee2-9ee7-64cd807d04bd (dataset), name: DatasetName { pool_name: ZpoolName { id: 60d88883-2ddc-48f7-92be-fb3130552398 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_c5e898ed-4202-4777-9b7c-9d08a9bd3c3a\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 64936e38-f87d-48f7-9fe9-a73f614a1c98 (dataset): DatasetConfig { id: 64936e38-f87d-48f7-9fe9-a73f614a1c98 (dataset), name: DatasetName { pool_name: ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6b4cad43-4f31-4ee6-baf3-34133fb352f9 (dataset): DatasetConfig { id: 6b4cad43-4f31-4ee6-baf3-34133fb352f9 (dataset), name: DatasetName { pool_name: ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }, kind: TransientZone { name: \"oxz_oximeter_13edfd02-a016-4abd-b9a3-b2af39dc85f9\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6e082723-189a-4ae8-ad1d-9c788e5ad902 (dataset): DatasetConfig { id: 6e082723-189a-4ae8-ad1d-9c788e5ad902 (dataset), name: DatasetName { pool_name: ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 725d02e7-ef0b-45b6-9daf-400d782153b0 (dataset): DatasetConfig { id: 725d02e7-ef0b-45b6-9daf-400d782153b0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 2735b386-4b94-446b-b6e4-69280ab9e40a (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 7eac0f17-2a31-4c63-84f2-30ad51a8c1bc (dataset): DatasetConfig { id: 7eac0f17-2a31-4c63-84f2-30ad51a8c1bc (dataset), name: DatasetName { pool_name: ZpoolName { id: d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7f63b141-11a6-4d53-8426-fe2d8c92530b (dataset): DatasetConfig { id: 7f63b141-11a6-4d53-8426-fe2d8c92530b (dataset), name: DatasetName { pool_name: ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 85304f96-cc43-4611-9315-f1ba7ead5e8b (dataset): DatasetConfig { id: 85304f96-cc43-4611-9315-f1ba7ead5e8b (dataset), name: DatasetName { pool_name: ZpoolName { id: d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_8796224f-ce1e-4f10-8b1b-63b2661798e0\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 86128a35-b205-4776-b2f1-2fa5df01c4b9 (dataset): DatasetConfig { id: 86128a35-b205-4776-b2f1-2fa5df01c4b9 (dataset), name: DatasetName { pool_name: ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_62b89797-eabf-4a54-ad84-6278e0d63bb2\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 87f7eea6-4e2b-4c9f-b8a7-1676f62d84ab (dataset): DatasetConfig { id: 87f7eea6-4e2b-4c9f-b8a7-1676f62d84ab (dataset), name: DatasetName { pool_name: ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8eb45c40-0d38-4946-828f-b00637a20c18 (dataset): DatasetConfig { id: 8eb45c40-0d38-4946-828f-b00637a20c18 (dataset), name: DatasetName { pool_name: ZpoolName { id: 2735b386-4b94-446b-b6e4-69280ab9e40a (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 96747428-6a4d-440e-90cd-3aa267ede24f (dataset): DatasetConfig { id: 96747428-6a4d-440e-90cd-3aa267ede24f (dataset), name: DatasetName { pool_name: ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 97fb4d68-ec04-4fff-9663-56b5a3dc9fae (dataset): DatasetConfig { id: 97fb4d68-ec04-4fff-9663-56b5a3dc9fae (dataset), name: DatasetName { pool_name: ZpoolName { id: 56581dc2-c967-4930-aa2d-7f607fddd15e (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 983df6bd-7882-42d2-8191-f3afaae6e762 (dataset): DatasetConfig { id: 983df6bd-7882-42d2-8191-f3afaae6e762 (dataset), name: DatasetName { pool_name: ZpoolName { id: 60d88883-2ddc-48f7-92be-fb3130552398 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_5d56a5dd-553d-43d1-ada2-7c96d8406842\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a1ff2dfd-9de8-46ee-b554-8b069dfb6df2 (dataset): DatasetConfig { id: a1ff2dfd-9de8-46ee-b554-8b069dfb6df2 (dataset), name: DatasetName { pool_name: ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a4fd30f1-4698-41b3-b8fc-cbb83226684e (dataset): DatasetConfig { id: a4fd30f1-4698-41b3-b8fc-cbb83226684e (dataset), name: DatasetName { pool_name: ZpoolName { id: 56581dc2-c967-4930-aa2d-7f607fddd15e (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_87ace5e0-9e49-47a5-9e60-8ef09cb36b5c\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a604712a-c938-4f89-b4b3-d4af9c23c9f8 (dataset): DatasetConfig { id: a604712a-c938-4f89-b4b3-d4af9c23c9f8 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_2a081fd5-0462-4328-9db6-071aae0d221b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b28a4670-0930-4cd4-b1e8-7bd1cc120b2b (dataset): DatasetConfig { id: b28a4670-0930-4cd4-b1e8-7bd1cc120b2b (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, b39c7a53-f320-4198-bafa-49bdea95c19a (dataset): DatasetConfig { id: b39c7a53-f320-4198-bafa-49bdea95c19a (dataset), name: DatasetName { pool_name: ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, b45db4e2-3947-420d-96cc-86dbebebf5e3 (dataset): DatasetConfig { id: b45db4e2-3947-420d-96cc-86dbebebf5e3 (dataset), name: DatasetName { pool_name: ZpoolName { id: 60d88883-2ddc-48f7-92be-fb3130552398 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b858f569-1fe5-42ba-a647-e3f0a40a44e9 (dataset): DatasetConfig { id: b858f569-1fe5-42ba-a647-e3f0a40a44e9 (dataset), name: DatasetName { pool_name: ZpoolName { id: e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, bc8bfb55-ea4b-4103-ab23-417c90b9a95b (dataset): DatasetConfig { id: bc8bfb55-ea4b-4103-ab23-417c90b9a95b (dataset), name: DatasetName { pool_name: ZpoolName { id: 4a60e634-3d93-4efd-bc48-2e2658b8d062 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, bda55e8e-d0c0-4309-bd71-c0a4c93a2ac9 (dataset): DatasetConfig { id: bda55e8e-d0c0-4309-bd71-c0a4c93a2ac9 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c01d9a75-c8ea-48b9-85a5-239afff33a22 (dataset): DatasetConfig { id: c01d9a75-c8ea-48b9-85a5-239afff33a22 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_e339542e-0ade-4538-9cba-05a7a02f2352\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c540951f-b03e-4936-a120-1d29ceab341d (dataset): DatasetConfig { id: c540951f-b03e-4936-a120-1d29ceab341d (dataset), name: DatasetName { pool_name: ZpoolName { id: 4a60e634-3d93-4efd-bc48-2e2658b8d062 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, c8467c06-cc64-41bb-a7dd-4bee12301547 (dataset): DatasetConfig { id: c8467c06-cc64-41bb-a7dd-4bee12301547 (dataset), name: DatasetName { pool_name: ZpoolName { id: 2735b386-4b94-446b-b6e4-69280ab9e40a (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_bac4f2f9-1ebe-4488-aa63-037e893be55f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cd3d1292-cc02-4be2-86c8-bed0e140083e (dataset): DatasetConfig { id: cd3d1292-cc02-4be2-86c8-bed0e140083e (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_852e9877-8d05-4fca-8f0b-0d193b0ac883\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cdd73160-8adf-4ae3-8284-d3a43d0bb817 (dataset): DatasetConfig { id: cdd73160-8adf-4ae3-8284-d3a43d0bb817 (dataset), name: DatasetName { pool_name: ZpoolName { id: d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cfda75af-330b-4e05-bb1c-ce25c890426b (dataset): DatasetConfig { id: cfda75af-330b-4e05-bb1c-ce25c890426b (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_c74c8efd-59c4-42a1-a06d-62bee6df73ab\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, dafd3791-da6e-44fc-877f-acce507145d6 (dataset): DatasetConfig { id: dafd3791-da6e-44fc-877f-acce507145d6 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, dc95eb60-7a86-4bec-8b0e-b0cc49c1c233 (dataset): DatasetConfig { id: dc95eb60-7a86-4bec-8b0e-b0cc49c1c233 (dataset), name: DatasetName { pool_name: ZpoolName { id: e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, dd1f66b6-28d5-4344-865a-a2d9989af950 (dataset): DatasetConfig { id: dd1f66b6-28d5-4344-865a-a2d9989af950 (dataset), name: DatasetName { pool_name: ZpoolName { id: e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_6130c690-a32a-4ce5-a932-f89d5df30ab6\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e52fbe4a-10a9-4e7f-8673-73b6bf967338 (dataset): DatasetConfig { id: e52fbe4a-10a9-4e7f-8673-73b6bf967338 (dataset), name: DatasetName { pool_name: ZpoolName { id: 4a60e634-3d93-4efd-bc48-2e2658b8d062 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_54430eae-0e39-408d-a34d-a9a9a575ad61\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ee752ced-3eca-4e7c-943c-78b64aaa6135 (dataset): DatasetConfig { id: ee752ced-3eca-4e7c-943c-78b64aaa6135 (dataset), name: DatasetName { pool_name: ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }, kind: TransientZone { name: \"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ef77125a-5142-4f6a-b05b-9e99a2ec57ca (dataset): DatasetConfig { id: ef77125a-5142-4f6a-b05b-9e99a2ec57ca (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fae30841-e06b-498a-af67-7a5ea9ae27b4 (dataset): DatasetConfig { id: fae30841-e06b-498a-af67-7a5ea9ae27b4 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_clickhouse_ad64b46c-b548-42d7-a128-733f9dbdafda\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fb00fe6c-d12b-4efd-9abd-e4bd4caab4d2 (dataset): DatasetConfig { id: fb00fe6c-d12b-4efd-9abd-e4bd4caab4d2 (dataset), name: DatasetName { pool_name: ZpoolName { id: 56581dc2-c967-4930-aa2d-7f607fddd15e (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_aa128c71-f260-4997-a503-cd00af8f00a9\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fcbf096a-f4db-4011-9059-2d8f549612a2 (dataset): DatasetConfig { id: fcbf096a-f4db-4011-9059-2d8f549612a2 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fe74d525-d726-4e22-a010-cea47f9145d3 (dataset): DatasetConfig { id: fe74d525-d726-4e22-a010-cea47f9145d3 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ff36f9bd-2e05-47da-8076-7fec6f106100 (dataset): DatasetConfig { id: ff36f9bd-2e05-47da-8076-7fec6f106100 (dataset), name: DatasetName { pool_name: ZpoolName { id: 56581dc2-c967-4930-aa2d-7f607fddd15e (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, fff1e4cc-ae24-4f16-b0c7-a3b33393e85a (dataset): DatasetConfig { id: fff1e4cc-ae24-4f16-b0c7-a3b33393e85a (dataset), name: DatasetName { pool_name: ZpoolName { id: 60d88883-2ddc-48f7-92be-fb3130552398 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }} }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.284794354Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"No ledger in /pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.284804104Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:189"}
{"msg":"No ledger in /pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.284811814Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:189"}
{"msg":"No previously-stored 'requested datasets', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.284818705Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:906"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.284824995Z","hostname":"sock","pid":1566,"name":"oxp_e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/zone/oxz_crucible_4fb64aa2-0fe1-47d4-b447-fc7e161481c6","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.284832355Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/zone/oxz_crucible_4fb64aa2-0fe1-47d4-b447-fc7e161481c6\"), full_name: \"oxp_e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/zone/oxz_crucible_4fb64aa2-0fe1-47d4-b447-fc7e161481c6\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.387481758Z","hostname":"sock","pid":1566,"name":"oxp_56581dc2-c967-4930-aa2d-7f607fddd15e/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.387533009Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_56581dc2-c967-4930-aa2d-7f607fddd15e/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.477776895Z","hostname":"sock","pid":1566,"name":"oxp_bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.477817886Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.567111888Z","hostname":"sock","pid":1566,"name":"oxp_56581dc2-c967-4930-aa2d-7f607fddd15e/crypt/zone/oxz_cockroachdb_23f9a8fe-7b99-4a86-9e17-b09185836552","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.567153488Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/56581dc2-c967-4930-aa2d-7f607fddd15e/crypt/zone/oxz_cockroachdb_23f9a8fe-7b99-4a86-9e17-b09185836552\"), full_name: \"oxp_56581dc2-c967-4930-aa2d-7f607fddd15e/crypt/zone/oxz_cockroachdb_23f9a8fe-7b99-4a86-9e17-b09185836552\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.667090818Z","hostname":"sock","pid":1566,"name":"oxp_4a60e634-3d93-4efd-bc48-2e2658b8d062/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.667132739Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/4a60e634-3d93-4efd-bc48-2e2658b8d062/crypt/zone\"), full_name: \"oxp_4a60e634-3d93-4efd-bc48-2e2658b8d062/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.735988078Z","hostname":"sock","pid":1566,"name":"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/zone/oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.736037488Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/zone/oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af\"), full_name: \"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/zone/oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.836347384Z","hostname":"sock","pid":1566,"name":"oxp_56581dc2-c967-4930-aa2d-7f607fddd15e/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.836389705Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_56581dc2-c967-4930-aa2d-7f607fddd15e/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.926270926Z","hostname":"sock","pid":1566,"name":"oxp_e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/zone/oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:15.926314316Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/zone/oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7\"), full_name: \"oxp_e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/zone/oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:16.028610603Z","hostname":"sock","pid":1566,"name":"oxp_e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:16.028653554Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab/crypt/zone\"), full_name: \"oxp_e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/df02b17d-a4bd-4ac6-9447-5abe2628a534)","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:16.08661173Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:16.097244958Z","hostname":"sock","pid":1566,"name":"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/zone/oxz_crucible_4a49f2d4-4a63-438e-9662-856d67052865","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:16.097269589Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/zone/oxz_crucible_4a49f2d4-4a63-438e-9662-856d67052865\"), full_name: \"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/zone/oxz_crucible_4a49f2d4-4a63-438e-9662-856d67052865\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:16.199153219Z","hostname":"sock","pid":1566,"name":"oxp_2735b386-4b94-446b-b6e4-69280ab9e40a/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:16.1991985Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_2735b386-4b94-446b-b6e4-69280ab9e40a/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:16.288142846Z","hostname":"sock","pid":1566,"name":"oxp_2735b386-4b94-446b-b6e4-69280ab9e40a/crypt/zone/oxz_nexus_4a05a39f-e9d2-455e-9cb5-f73902d6694b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:16.288190097Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/2735b386-4b94-446b-b6e4-69280ab9e40a/crypt/zone/oxz_nexus_4a05a39f-e9d2-455e-9cb5-f73902d6694b\"), full_name: \"oxp_2735b386-4b94-446b-b6e4-69280ab9e40a/crypt/zone/oxz_nexus_4a05a39f-e9d2-455e-9cb5-f73902d6694b\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:16.389825204Z","hostname":"sock","pid":1566,"name":"oxp_d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:16.389870535Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1/crypt/debug\"), full_name: \"oxp_d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:16.458761114Z","hostname":"sock","pid":1566,"name":"oxp_e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:16.458803424Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:16.55217989Z","hostname":"sock","pid":1566,"name":"oxp_60d88883-2ddc-48f7-92be-fb3130552398/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:16.552218571Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/60d88883-2ddc-48f7-92be-fb3130552398/crypt/zone\"), full_name: \"oxp_60d88883-2ddc-48f7-92be-fb3130552398/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:16.620914017Z","hostname":"sock","pid":1566,"name":"oxp_e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:16.620955988Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/zone\"), full_name: \"oxp_e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:16.690222813Z","hostname":"sock","pid":1566,"name":"oxp_bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crypt/zone/oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:16.690265814Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crypt/zone/oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7\"), full_name: \"oxp_bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crypt/zone/oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:16.787895627Z","hostname":"sock","pid":1566,"name":"oxp_bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:16.787933448Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:16.875865198Z","hostname":"sock","pid":1566,"name":"oxp_56581dc2-c967-4930-aa2d-7f607fddd15e/crypt/zone/oxz_crucible_1a248290-007e-4b53-b1f1-6f7a9e3dbb71","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:16.875906189Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/56581dc2-c967-4930-aa2d-7f607fddd15e/crypt/zone/oxz_crucible_1a248290-007e-4b53-b1f1-6f7a9e3dbb71\"), full_name: \"oxp_56581dc2-c967-4930-aa2d-7f607fddd15e/crypt/zone/oxz_crucible_1a248290-007e-4b53-b1f1-6f7a9e3dbb71\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:16.973143565Z","hostname":"sock","pid":1566,"name":"oxp_e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/zone/oxz_cockroachdb_9e92e318-bc88-437a-bf0e-cb6cc571e7b1","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:16.973184606Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/zone/oxz_cockroachdb_9e92e318-bc88-437a-bf0e-cb6cc571e7b1\"), full_name: \"oxp_e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/zone/oxz_cockroachdb_9e92e318-bc88-437a-bf0e-cb6cc571e7b1\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:17.072596517Z","hostname":"sock","pid":1566,"name":"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/zone/oxz_cockroachdb_6ca40c88-6f5e-4727-8259-0993d419cff1","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:17.072642358Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/zone/oxz_cockroachdb_6ca40c88-6f5e-4727-8259-0993d419cff1\"), full_name: \"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/zone/oxz_cockroachdb_6ca40c88-6f5e-4727-8259-0993d419cff1\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/df02b17d-a4bd-4ac6-9447-5abe2628a534)","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:17.087442422Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:17.173196008Z","hostname":"sock","pid":1566,"name":"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:17.173230788Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:17.265484857Z","hostname":"sock","pid":1566,"name":"oxp_60d88883-2ddc-48f7-92be-fb3130552398/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:17.265523767Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/60d88883-2ddc-48f7-92be-fb3130552398/crypt/debug\"), full_name: \"oxp_60d88883-2ddc-48f7-92be-fb3130552398/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:17.334148683Z","hostname":"sock","pid":1566,"name":"oxp_60d88883-2ddc-48f7-92be-fb3130552398/crypt/zone/oxz_cockroachdb_710d532b-120e-4fdc-9cae-a0252c955c30","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:17.334200043Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/60d88883-2ddc-48f7-92be-fb3130552398/crypt/zone/oxz_cockroachdb_710d532b-120e-4fdc-9cae-a0252c955c30\"), full_name: \"oxp_60d88883-2ddc-48f7-92be-fb3130552398/crypt/zone/oxz_cockroachdb_710d532b-120e-4fdc-9cae-a0252c955c30\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:17.432135922Z","hostname":"sock","pid":1566,"name":"oxp_bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crypt/zone/oxz_external_dns_5267fc4d-d51f-4cd0-9820-0c103fa42eea","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:17.432176522Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crypt/zone/oxz_external_dns_5267fc4d-d51f-4cd0-9820-0c103fa42eea\"), full_name: \"oxp_bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crypt/zone/oxz_external_dns_5267fc4d-d51f-4cd0-9820-0c103fa42eea\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:17.532769853Z","hostname":"sock","pid":1566,"name":"oxp_60d88883-2ddc-48f7-92be-fb3130552398/crypt/zone/oxz_crucible_c5e898ed-4202-4777-9b7c-9d08a9bd3c3a","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:17.532814483Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/60d88883-2ddc-48f7-92be-fb3130552398/crypt/zone/oxz_crucible_c5e898ed-4202-4777-9b7c-9d08a9bd3c3a\"), full_name: \"oxp_60d88883-2ddc-48f7-92be-fb3130552398/crypt/zone/oxz_crucible_c5e898ed-4202-4777-9b7c-9d08a9bd3c3a\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:17.631786908Z","hostname":"sock","pid":1566,"name":"oxp_e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:17.631827279Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:17.720222656Z","hostname":"sock","pid":1566,"name":"oxp_e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/zone/oxz_oximeter_13edfd02-a016-4abd-b9a3-b2af39dc85f9","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:17.720267997Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/zone/oxz_oximeter_13edfd02-a016-4abd-b9a3-b2af39dc85f9\"), full_name: \"oxp_e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/zone/oxz_oximeter_13edfd02-a016-4abd-b9a3-b2af39dc85f9\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:17.819908202Z","hostname":"sock","pid":1566,"name":"oxp_e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:17.819957413Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/debug\"), full_name: \"oxp_e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:17.888641239Z","hostname":"sock","pid":1566,"name":"oxp_2735b386-4b94-446b-b6e4-69280ab9e40a/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:17.88868397Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/2735b386-4b94-446b-b6e4-69280ab9e40a/crypt/debug\"), full_name: \"oxp_2735b386-4b94-446b-b6e4-69280ab9e40a/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:17.955995634Z","hostname":"sock","pid":1566,"name":"oxp_d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:17.956035855Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:18.044570394Z","hostname":"sock","pid":1566,"name":"oxp_bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:18.044601775Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crypt/external_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/df02b17d-a4bd-4ac6-9447-5abe2628a534)","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:18.090014783Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:18.135003694Z","hostname":"sock","pid":1566,"name":"oxp_d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1/crypt/zone/oxz_crucible_8796224f-ce1e-4f10-8b1b-63b2661798e0","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:18.135048455Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1/crypt/zone/oxz_crucible_8796224f-ce1e-4f10-8b1b-63b2661798e0\"), full_name: \"oxp_d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1/crypt/zone/oxz_crucible_8796224f-ce1e-4f10-8b1b-63b2661798e0\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:18.234842252Z","hostname":"sock","pid":1566,"name":"oxp_e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/zone/oxz_crucible_pantry_62b89797-eabf-4a54-ad84-6278e0d63bb2","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:18.234895733Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/zone/oxz_crucible_pantry_62b89797-eabf-4a54-ad84-6278e0d63bb2\"), full_name: \"oxp_e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/zone/oxz_crucible_pantry_62b89797-eabf-4a54-ad84-6278e0d63bb2\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:18.33403524Z","hostname":"sock","pid":1566,"name":"oxp_bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:18.334079481Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crypt/zone\"), full_name: \"oxp_bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:18.401765051Z","hostname":"sock","pid":1566,"name":"oxp_2735b386-4b94-446b-b6e4-69280ab9e40a/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:18.401802392Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/2735b386-4b94-446b-b6e4-69280ab9e40a/crypt/zone\"), full_name: \"oxp_2735b386-4b94-446b-b6e4-69280ab9e40a/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:18.470819333Z","hostname":"sock","pid":1566,"name":"oxp_bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:18.470856844Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:18.563754962Z","hostname":"sock","pid":1566,"name":"oxp_56581dc2-c967-4930-aa2d-7f607fddd15e/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:18.563801973Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/56581dc2-c967-4930-aa2d-7f607fddd15e/crypt/zone\"), full_name: \"oxp_56581dc2-c967-4930-aa2d-7f607fddd15e/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:18.630780012Z","hostname":"sock","pid":1566,"name":"oxp_60d88883-2ddc-48f7-92be-fb3130552398/crypt/zone/oxz_nexus_5d56a5dd-553d-43d1-ada2-7c96d8406842","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:18.630841613Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/60d88883-2ddc-48f7-92be-fb3130552398/crypt/zone/oxz_nexus_5d56a5dd-553d-43d1-ada2-7c96d8406842\"), full_name: \"oxp_60d88883-2ddc-48f7-92be-fb3130552398/crypt/zone/oxz_nexus_5d56a5dd-553d-43d1-ada2-7c96d8406842\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:18.72996416Z","hostname":"sock","pid":1566,"name":"oxp_e858b5e9-aa1b-415e-9d0d-1b75991b7849/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:18.730011081Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_e858b5e9-aa1b-415e-9d0d-1b75991b7849/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:18.816461548Z","hostname":"sock","pid":1566,"name":"oxp_56581dc2-c967-4930-aa2d-7f607fddd15e/crypt/zone/oxz_nexus_87ace5e0-9e49-47a5-9e60-8ef09cb36b5c","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:18.816502928Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/56581dc2-c967-4930-aa2d-7f607fddd15e/crypt/zone/oxz_nexus_87ace5e0-9e49-47a5-9e60-8ef09cb36b5c\"), full_name: \"oxp_56581dc2-c967-4930-aa2d-7f607fddd15e/crypt/zone/oxz_nexus_87ace5e0-9e49-47a5-9e60-8ef09cb36b5c\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:18.913834167Z","hostname":"sock","pid":1566,"name":"oxp_bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crypt/zone/oxz_crucible_2a081fd5-0462-4328-9db6-071aae0d221b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:18.913880458Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crypt/zone/oxz_crucible_2a081fd5-0462-4328-9db6-071aae0d221b\"), full_name: \"oxp_bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crypt/zone/oxz_crucible_2a081fd5-0462-4328-9db6-071aae0d221b\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:19.012483606Z","hostname":"sock","pid":1566,"name":"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:19.012535637Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/debug\"), full_name: \"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:19.080240937Z","hostname":"sock","pid":1566,"name":"oxp_bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:19.080288698Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crypt/debug\"), full_name: \"oxp_bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/df02b17d-a4bd-4ac6-9447-5abe2628a534)","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:19.091930072Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:19.149577373Z","hostname":"sock","pid":1566,"name":"oxp_60d88883-2ddc-48f7-92be-fb3130552398/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:19.149615984Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_60d88883-2ddc-48f7-92be-fb3130552398/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:19.240771525Z","hostname":"sock","pid":1566,"name":"oxp_e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:19.240818046Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab/crypt/debug\"), full_name: \"oxp_e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:19.30751005Z","hostname":"sock","pid":1566,"name":"oxp_4a60e634-3d93-4efd-bc48-2e2658b8d062/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:19.307545531Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_4a60e634-3d93-4efd-bc48-2e2658b8d062/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:19.394527046Z","hostname":"sock","pid":1566,"name":"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:19.394565497Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/external_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:19.483071596Z","hostname":"sock","pid":1566,"name":"oxp_bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crypt/zone/oxz_cockroachdb_e339542e-0ade-4538-9cba-05a7a02f2352","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:19.483125647Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crypt/zone/oxz_cockroachdb_e339542e-0ade-4538-9cba-05a7a02f2352\"), full_name: \"oxp_bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4/crypt/zone/oxz_cockroachdb_e339542e-0ade-4538-9cba-05a7a02f2352\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:19.581761576Z","hostname":"sock","pid":1566,"name":"oxp_4a60e634-3d93-4efd-bc48-2e2658b8d062/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:19.581805897Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/4a60e634-3d93-4efd-bc48-2e2658b8d062/crypt/debug\"), full_name: \"oxp_4a60e634-3d93-4efd-bc48-2e2658b8d062/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:19.64964111Z","hostname":"sock","pid":1566,"name":"oxp_2735b386-4b94-446b-b6e4-69280ab9e40a/crypt/zone/oxz_crucible_bac4f2f9-1ebe-4488-aa63-037e893be55f","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:19.64968214Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/2735b386-4b94-446b-b6e4-69280ab9e40a/crypt/zone/oxz_crucible_bac4f2f9-1ebe-4488-aa63-037e893be55f\"), full_name: \"oxp_2735b386-4b94-446b-b6e4-69280ab9e40a/crypt/zone/oxz_crucible_bac4f2f9-1ebe-4488-aa63-037e893be55f\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:19.748792747Z","hostname":"sock","pid":1566,"name":"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/zone/oxz_crucible_pantry_852e9877-8d05-4fca-8f0b-0d193b0ac883","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:19.748841708Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/zone/oxz_crucible_pantry_852e9877-8d05-4fca-8f0b-0d193b0ac883\"), full_name: \"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/zone/oxz_crucible_pantry_852e9877-8d05-4fca-8f0b-0d193b0ac883\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:19.947411547Z","hostname":"sock","pid":1566,"name":"oxp_d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:19.947463988Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1/crypt/zone\"), full_name: \"oxp_d7f8995a-74b6-4a9d-b142-4b7ddbdf63e1/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:20.044106456Z","hostname":"sock","pid":1566,"name":"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/zone/oxz_external_dns_c74c8efd-59c4-42a1-a06d-62bee6df73ab","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:20.044161437Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/zone/oxz_external_dns_c74c8efd-59c4-42a1-a06d-62bee6df73ab\"), full_name: \"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/zone/oxz_external_dns_c74c8efd-59c4-42a1-a06d-62bee6df73ab\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/df02b17d-a4bd-4ac6-9447-5abe2628a534)","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:20.093587128Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:20.198775941Z","hostname":"sock","pid":1566,"name":"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/clickhouse","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:20.198814692Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/clickhouse\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:20.332551666Z","hostname":"sock","pid":1566,"name":"oxp_e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:20.332591697Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:20.471656765Z","hostname":"sock","pid":1566,"name":"oxp_e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab/crypt/zone/oxz_crucible_6130c690-a32a-4ce5-a932-f89d5df30ab6","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:20.471704036Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab/crypt/zone/oxz_crucible_6130c690-a32a-4ce5-a932-f89d5df30ab6\"), full_name: \"oxp_e5bb58f6-a9bf-4899-acd4-a32e9b6d18ab/crypt/zone/oxz_crucible_6130c690-a32a-4ce5-a932-f89d5df30ab6\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:20.609520285Z","hostname":"sock","pid":1566,"name":"oxp_4a60e634-3d93-4efd-bc48-2e2658b8d062/crypt/zone/oxz_crucible_54430eae-0e39-408d-a34d-a9a9a575ad61","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:20.609570156Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/4a60e634-3d93-4efd-bc48-2e2658b8d062/crypt/zone/oxz_crucible_54430eae-0e39-408d-a34d-a9a9a575ad61\"), full_name: \"oxp_4a60e634-3d93-4efd-bc48-2e2658b8d062/crypt/zone/oxz_crucible_54430eae-0e39-408d-a34d-a9a9a575ad61\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:20.740364154Z","hostname":"sock","pid":1566,"name":"oxp_e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/zone/oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:20.740422655Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/zone/oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e\"), full_name: \"oxp_e858b5e9-aa1b-415e-9d0d-1b75991b7849/crypt/zone/oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:20.874969852Z","hostname":"sock","pid":1566,"name":"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:20.875015772Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/zone\"), full_name: \"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:20.944998639Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2919013623316299948"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:20.945131351Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5126378412384038234"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:20.964310554Z","hostname":"sock","pid":1566,"name":"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/zone/oxz_clickhouse_ad64b46c-b548-42d7-a128-733f9dbdafda","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:20.964351325Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/zone/oxz_clickhouse_ad64b46c-b548-42d7-a128-733f9dbdafda\"), full_name: \"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/zone/oxz_clickhouse_ad64b46c-b548-42d7-a128-733f9dbdafda\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.043981843Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4350"}
{"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.044038024Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4360","gateway":"fd00:1122:3344:101::1"}
{"msg":"Default route already exists","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.05011182Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4374","gateway":"fd00:1122:3344:101::1"}
{"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.05014302Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4439"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/df02b17d-a4bd-4ac6-9447-5abe2628a534)","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:21.098071188Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.116267856Z","hostname":"sock","pid":1566,"name":"oxp_56581dc2-c967-4930-aa2d-7f607fddd15e/crypt/zone/oxz_crucible_pantry_aa128c71-f260-4997-a503-cd00af8f00a9","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.116307676Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/56581dc2-c967-4930-aa2d-7f607fddd15e/crypt/zone/oxz_crucible_pantry_aa128c71-f260-4997-a503-cd00af8f00a9\"), full_name: \"oxp_56581dc2-c967-4930-aa2d-7f607fddd15e/crypt/zone/oxz_crucible_pantry_aa128c71-f260-4997-a503-cd00af8f00a9\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.196626386Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4505"}
{"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.196686667Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4532"}
{"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.238244204Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4545"}
{"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.238280615Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4391"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.263789888Z","hostname":"sock","pid":1566,"name":"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.263845439Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:21.278398679Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"12.897827825s"}
{"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.293183633Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4433"}
{"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.293214053Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4512","rack_subnet":"fd00:1122:3344::"}
{"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.314042323Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4523"}
{"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.314082913Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4565"}
{"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.385093466Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4597"}
{"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.385134307Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4603"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.400181034Z","hostname":"sock","pid":1566,"name":"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.400216075Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_7114ae1a-4fea-4be4-be38-9a0f250ad6f4/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.476281697Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4644"}
{"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.476334598Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:370"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/df02b17d-a4bd-4ac6-9447-5abe2628a534)","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:21.514489021Z","hostname":"sock","pid":1566,"component":"NexusNotifierTask","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/nexus.rs:530"}
{"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.517812194Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:418"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/df02b17d-a4bd-4ac6-9447-5abe2628a534)","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:21.51881732Z","hostname":"sock","pid":1566,"component":"NexusNotifierTask","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/nexus.rs:530"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.536891726Z","hostname":"sock","pid":1566,"name":"oxp_56581dc2-c967-4930-aa2d-7f607fddd15e/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.536903906Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/56581dc2-c967-4930-aa2d-7f607fddd15e/crypt/debug\"), full_name: \"oxp_56581dc2-c967-4930-aa2d-7f607fddd15e/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"}
{"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.556645568Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:686"}
{"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.55676523Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:731"}
{"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.557187276Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:734","uptime":"1"}
{"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.557203137Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:440","config":"PortSettings {\n    links: {\n        \"0\": LinkSettings {\n            addrs: [\n                10.151.2.174,\n            ],\n            params: LinkCreate {\n                autoneg: false,\n                fec: Some(\n                    None,\n                ),\n                kr: false,\n                lane: Some(\n                    LinkId(\n                        0,\n                    ),\n                ),\n                speed: Speed40G,\n                tx_eq: None,\n            },\n        },\n    },\n}"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.606871232Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4177"}
{"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.629617432Z","hostname":"sock","pid":1566,"name":"oxp_60d88883-2ddc-48f7-92be-fb3130552398/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"}
{"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.629673542Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_60d88883-2ddc-48f7-92be-fb3130552398/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"}
{"msg":"Writing ledger to /pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/.omicron-datasets.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.757326021Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:200"}
{"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.757675706Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/omicron-datasets.json\""}
{"msg":"Writing ledger to /pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/.omicron-datasets.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.757707737Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:200"}
{"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.757973901Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/omicron-datasets.json\""}
{"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.758068023Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.758107213Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.758119633Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.758127583Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.758133204Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.758140394Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.758146014Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.758151574Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.758158364Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.758163524Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.758169184Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.758181704Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.758188604Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.758192554Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.758199665Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.758203895Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.758209405Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.758214525Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.758219725Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.758225065Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.758230335Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.758235625Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.758240905Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"datasets ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.758251475Z","hostname":"sock","pid":1566,"sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:852"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.758339247Z","hostname":"sock","pid":1566,"uri":"/datasets","method":"PUT","req_id":"7c38eae3-289d-4638-9a74-75e955b590b7","remote_addr":"[fd00:1122:3344:101::1]:39695","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6474306,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 820116a3-d9c1-4adb-873f-47498c0c38af (service), filesystem_pool: Some(ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 34c52a54-f90d-4998-a35f-4abe932744d7 (service), filesystem_pool: Some(ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 32d08044-1c99-436c-a7a8-db8e905f59e7 (service), filesystem_pool: Some(ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.791124635Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.791257687Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:55279"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.791527261Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"No ledger in /pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.791566112Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:189"}
{"msg":"No ledger in /pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.791572352Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:189"}
{"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.791627803Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Reading ledger from /pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.791635253Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"}
{"msg":"Reading ledger from /pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.791835396Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"}
{"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.791981089Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.7920803Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.836511533Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.836547263Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.836551343Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.886575404Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.890148521Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.893866979Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"}
{"msg":"Installing Omicron zone: oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.90589691Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"}
{"msg":"Installing Omicron zone: oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.910109546Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"}
{"msg":"Installing Omicron zone: oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:21.914026688Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/df02b17d-a4bd-4ac6-9447-5abe2628a534)","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:22.100210701Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:22.300485317Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:22.300589969Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"Profile for oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService1'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:2::2'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:2::1'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:2::1]:5353'/>\n        <propval type=\"astring\" name=\"dns_address\" value='[fd00:1122:3344:2::1]:53'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:23.51769791Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:23.517837312Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"}
{"msg":"Booting oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:23.518040746Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService2'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:1::2'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:1::1'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:1::1]:5353'/>\n        <propval type=\"astring\" name=\"dns_address\" value='[fd00:1122:3344:1::1]:53'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:24.004078759Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService3'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:3::2'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:3::1'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:3::1]:5353'/>\n        <propval type=\"astring\" name=\"dns_address\" value='[fd00:1122:3344:3::1]:53'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:24.482586684Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:24.482679206Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:24.482695616Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"}
{"msg":"wait for service Some(\"oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7\") failed: Property not found. retry in 39.021272ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:24.491361833Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Booting oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:24.491416154Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Booting oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:24.495509279Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7\") failed: Property not found. retry in 124.21638ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:24.539604706Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af\") failed: Property not found. retry in 26.1756ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:24.620415043Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7\") failed: Property not found. retry in 43.887595ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:24.626774464Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af\") failed: Property not found. retry in 136.789524ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:24.654526053Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7\") failed: Property not found. retry in 158.551508ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:24.672436256Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7\") failed: Property not found. retry in 61.838308ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:24.68029352Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7\") failed: Property not found. retry in 250.12968ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:24.9541556Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7\") failed: Property not found. retry in 444.451039ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:25.017064204Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af\") failed: Property not found. retry in 208.317613ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:25.044338005Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7\") failed: Property not found. retry in 211.827618ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:25.217843548Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af\") failed: Property not found. retry in 516.479456ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:25.26355201Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7\") failed: Property not found. retry in 746.487162ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:25.438551547Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7\") failed: Property not found. retry in 1.07586158s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:25.470678375Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af\") failed: Property not found. retry in 1.161320863s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:25.789237012Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7\") failed: Property not found. retry in 1.488956176s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:26.194255385Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7\") failed: Property not found. retry in 594.004998ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:26.557115352Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af\") failed: Property not found. retry in 570.453246ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:26.961534945Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7\") failed: Property not found. retry in 1.298285377s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:27.160891536Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af\") failed: Property not found. retry in 1.0586591s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:27.541690886Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7\") failed: Property not found. retry in 879.697538ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:27.691387993Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7\") failed: Property not found. retry in 782.526774ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:28.467510473Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7\") failed: Property not found. retry in 1.049641327s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:28.578247604Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af\") failed: Property not found. retry in 1.098118686s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:28.608144046Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7\") failed: Property not found. retry in 1.365018532s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:29.259248889Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7\") failed: Property not found. retry in 1.15728784s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:29.636532464Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af\") failed: Property not found. retry in 1.068020006s","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:29.714851813Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:30.814588958Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_internal_dns_34c52a54-f90d-4998-a35f-4abe932744d7"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:30.814641689Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_internal_dns_820116a3-d9c1-4adb-873f-47498c0c38af"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:30.814650319Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_internal_dns_32d08044-1c99-436c-a7a8-db8e905f59e7"}
{"msg":"Writing ledger to /pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:30.814661569Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:200"}
{"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:30.814960814Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/omicron-zones.json\""}
{"msg":"Writing ledger to /pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:30.814972404Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:200"}
{"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:30.815175557Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/omicron-zones.json\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:30.815259949Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"34944634-971c-45ff-aa7a-af758e8f4d09","remote_addr":"[fd00:1122:3344:101::1]:55279","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9023899,"response_code":"204"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:30.818224856Z","hostname":"sock","pid":1566,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:30.946300941Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2919013623316299948"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:30.946421812Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5126378412384038234"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.021926084Z","hostname":"sock","pid":1566,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.096205529Z","hostname":"sock","pid":1566,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655"}
{"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.169792262Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:691"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.169808922Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:264"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.200570229Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.201475113Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:306"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.201483203Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 820116a3-d9c1-4adb-873f-47498c0c38af (service), filesystem_pool: Some(ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7114ae1a-4fea-4be4-be38-9a0f250ad6f4 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 34c52a54-f90d-4998-a35f-4abe932744d7 (service), filesystem_pool: Some(ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc8b77fc-ded9-497e-9dd4-28fb6a7b1fb4 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 32d08044-1c99-436c-a7a8-db8e905f59e7 (service), filesystem_pool: Some(ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71e73077-bc8e-4f9f-8689-878cda8ba43e (service), filesystem_pool: Some(ZpoolName { id: e858b5e9-aa1b-415e-9d0d-1b75991b7849 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b11bb979-bfa7-42d1-abb4-110ef60451a7, kind: Service { id: 71e73077-bc8e-4f9f-8689-878cda8ba43e }, name: Name(\"ntp-71e73077-bc8e-4f9f-8689-878cda8ba43e\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 147, 44])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.231093091Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.231181813Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:59783"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.231374436Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Reading ledger from /pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.231396646Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"}
{"msg":"Reading ledger from /pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.231522538Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:178"}
{"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.231595259Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Reading ledger from /pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.231608699Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"}
{"msg":"Reading ledger from /pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.231780342Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"}
{"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.231921284Z","hostname":"sock","pid":1566,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.231971955Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.245358517Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.27906513Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.279790131Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.279795421Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:32.302986618Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:32.30310253Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.309547332Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 147, 44]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.309557322Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"boundary_ntp"}
{"msg":"no nat entry found for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.309984799Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"}
{"msg":"creating nat entry for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.309991359Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"}
{"msg":"creation of nat entry successful for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.310272813Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.31072419Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"}
{"msg":"Configuring new Omicron zone: oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.334398254Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"}
{"msg":"Installing Omicron zone: oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.352771965Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"}
{"msg":"Profile for oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService4'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::1b'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/chrony-setup\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"allow\" value='fd00:1122:3344:100::/56'/>\n        <propval type=\"boolean\" name=\"boundary\" value='true'/>\n        <propval type=\"astring\" name=\"boundary_pool\" value='boundary-ntp.control-plane.oxide.internal'/>\n        <propval type=\"astring\" name=\"server\" value='0.pool.ntp.org'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"1.1.1.1\" />\n            <value_node value=\"9.9.9.9\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/ntp\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value='opte0'/>\n        <propval type=\"astring\" name=\"gateway\" value='172.30.3.1'/>\n        <propval type=\"astring\" name=\"ip\" value='172.30.3.5'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.977594853Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:32.977742205Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e\") failed: Property not found. retry in 35.165544ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:33.111326507Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e\") failed: Property not found. retry in 61.505553ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:33.154034502Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e\") failed: Property not found. retry in 278.196134ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:33.227086997Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e\") failed: Property not found. retry in 241.548094ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:33.517794903Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e\") failed: Property not found. retry in 794.809744ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:33.768569498Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e\") failed: Property not found. retry in 961.803412ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:34.572436077Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e\") failed: Property not found. retry in 556.42732ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:35.542367361Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e\") failed: Property not found. retry in 882.640032ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:36.10782816Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e\") failed: Property not found. retry in 870.561485ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:37.000192348Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e\") failed: Property not found. retry in 564.733808ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:37.879953947Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e\") failed: Property not found. retry in 610.306145ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:38.454022032Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e\") failed: Property not found. retry in 614.551044ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:39.073630288Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:39.708324462Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3318","zone":"oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e"}
{"msg":"Writing ledger to /pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:39.708377833Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:200"}
{"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:39.708760449Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/67a39b8e-1dc3-40e2-8b0f-59c429bf4f91/config/omicron-zones.json\""}
{"msg":"Writing ledger to /pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:39.708793559Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:200"}
{"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:39.709028023Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/4c94aa86-6837-420b-a3c1-83899731e9b0/config/omicron-zones.json\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:39.709119284Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"36b6c56d-9a7c-4991-a431-18e9e4b36947","remote_addr":"[fd00:1122:3344:101::1]:59783","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7477873,"response_code":"204"}
{"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:39.711922809Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:732"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:39.742103546Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:39.742253828Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:62845"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:39.751238901Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3958"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:39.751346602Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"f868388e-d7c6-4bd9-8156-a9165bd61975","remote_addr":"[fd00:1122:3344:101::1]:62845","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8983,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:39.754143567Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:39.916251079Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:39.916391012Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:37041"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:40.647657982Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5398698369582487023"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:40.812760722Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15858676633825570490"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:40.816088685Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2459447259745793341"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:40.947408991Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2919013623316299948"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:40.947678685Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5126378412384038234"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:42.305006183Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:42.305123356Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:43.925424041Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3958"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:43.925563914Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"5a632b47-80d6-438f-8258-b70940845120","remote_addr":"[fd00:1122:3344:101::1]:37041","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009104,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:43.928333647Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:44.04875755Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:44.048960074Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:54879"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:44.057675251Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3958"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:44.057811394Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"46559ed5-a03f-47fd-b980-556fefa183d7","remote_addr":"[fd00:1122:3344:101::1]:54879","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8766,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:44.061154317Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:44.187326431Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:44.187481474Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:52517"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:48.195954886Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3958"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:48.196016037Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"6db0970d-cef1-427a-bca3-e2f5112804b8","remote_addr":"[fd00:1122:3344:101::1]:52517","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008474,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:48.199187007Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:48.321882447Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:48.32206662Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:53316"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:48.331230805Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"4a5ad9b6-b8c7-40d3-b4a8-9919f741b0e0","remote_addr":"[fd00:1122:3344:101::1]:53316","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9090,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:48.334589707Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:48.334598418Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:48.495757886Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:48.495902048Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:57812"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:48.504357982Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"9fcc6454-8954-47f2-97ab-8fb65d03b6b1","remote_addr":"[fd00:1122:3344:101::1]:57812","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8351,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:48.507811706Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:48.507819466Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:48.827927427Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:48.828046319Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:35679"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:48.836091176Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"4374df11-8ce4-4b69-8607-fe67ba47682b","remote_addr":"[fd00:1122:3344:101::1]:35679","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7992,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:48.838190289Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:48.838199149Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:49.099074074Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:49.099205906Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:62024"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:49.106907758Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"8fbc985c-4594-44a5-94c5-6a8e6f775d67","remote_addr":"[fd00:1122:3344:101::1]:62024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7648,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:49.109643361Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:49.109670011Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:49.214742482Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"174.186421ms"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:49.660083223Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:49.660200655Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:48928"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:49.668186091Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"84a7cacf-c51e-4cee-b6e0-432b11a4f932","remote_addr":"[fd00:1122:3344:101::1]:48928","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7939,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:49.670304225Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:49.670310595Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:49.711864582Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"329491034640178752"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:49.711991934Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17995928283259856397"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:50.052016109Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:50.052167321Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:46930"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:50.060655955Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"d2ab2c65-902f-4fde-bc53-87b2dad0fe76","remote_addr":"[fd00:1122:3344:101::1]:46930","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8419,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:50.062266951Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:50.062272111Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:50.603348955Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:50.603508928Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:51984"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:50.612184325Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"c042cb54-aa8e-455d-a9c9-c466e581b1ed","remote_addr":"[fd00:1122:3344:101::1]:51984","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8574,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:50.614298879Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:50.614306039Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:50.64981263Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5398698369582487023"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:50.81487215Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15858676633825570490"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:50.817146095Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2459447259745793341"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:50.949435177Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2919013623316299948"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:50.949663701Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5126378412384038234"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:51.256237307Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:33590"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:51.2564335Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"e4b74605-12e0-4384-9e67-5038221badcd","remote_addr":"[fdb0:200::2]:33590","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89,"response_code":"200"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:51.368559442Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:51.368705615Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:38936"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:51.377141928Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"b153d715-c3c5-4786-96d4-04dafdf56152","remote_addr":"[fd00:1122:3344:101::1]:38936","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8347,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:51.379382094Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:51.379388674Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:51.857208948Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:51.85735703Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:40778"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:51.865372947Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"be7a0519-585a-478e-a4b8-bbbbaaecf5fd","remote_addr":"[fd00:1122:3344:101::1]:40778","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7951,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:51.868212982Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:51.868228742Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:52.306701834Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:52.306823296Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:52.806695408Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:52.806846271Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:64128"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:52.815334245Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"2b46072f-012a-4652-949c-75a2d0ca4d18","remote_addr":"[fd00:1122:3344:101::1]:64128","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8420,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:52.819331718Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:52.819342548Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:53.732657358Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:53.73282608Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:34240"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:53.741233023Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"1233fd8e-5c30-49b4-a3d1-e31c739bbf59","remote_addr":"[fd00:1122:3344:101::1]:34240","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8338,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:53.745155585Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:53.745183436Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:55.635383439Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:55.635566922Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:45934"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:55.643824362Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"a92329e0-b835-4545-bf2f-55382dcdd844","remote_addr":"[fd00:1122:3344:101::1]:45934","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8179,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:55.646109749Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:55.646120158Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:57.073680897Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:57.07383331Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:37674"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:57.083136607Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"8a0b64bb-72db-4042-a04b-954492606962","remote_addr":"[fd00:1122:3344:101::1]:37674","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9228,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:57.087503856Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:57.087515546Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:58.226508743Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:58.226690575Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:50092"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:58.235130059Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"cb10acf4-a09f-4db2-bef1-1e59541c2243","remote_addr":"[fd00:1122:3344:101::1]:50092","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8350,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:10:58.237372044Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:10:58.237378084Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:59.713997169Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"329491034640178752"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:10:59.714112761Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17995928283259856397"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:00.651531601Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5398698369582487023"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:00.81655625Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15858676633825570490"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:00.817687818Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2459447259745793341"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:00.950988365Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2919013623316299948"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:00.951212789Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5126378412384038234"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:01.019973775Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:01.020100427Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:39891"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:01.028210105Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"53e2de96-fa4d-444d-8c0f-52ab9e0dd533","remote_addr":"[fd00:1122:3344:101::1]:39891","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7992,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:01.03037228Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:11:01.03037918Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:02.30937972Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:02.309502602Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:03.582822302Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:03.583010095Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:54504"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:03.591818004Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"60c45d60-e451-4ba7-8a47-1ec572653850","remote_addr":"[fd00:1122:3344:101::1]:54504","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8723,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:03.594117171Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:11:03.594121971Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:11:04.42392551Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"323.798286ms"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:07.295636749Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:07.295747591Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:50673"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:07.304580301Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"0feb1384-8862-4187-95f0-4cfa331e96f6","remote_addr":"[fd00:1122:3344:101::1]:50673","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8638,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:07.307350975Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:11:07.307357825Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:09.716130036Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"329491034640178752"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:09.716253018Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17995928283259856397"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:10.653596967Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5398698369582487023"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:10.818533664Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15858676633825570490"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:10.818777898Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2459447259745793341"}
{"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:10.951939533Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"2919013623316299948"}
{"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:10.952495412Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"5126378412384038234"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:12.311813921Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:12.311927303Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:12.413107683Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:12.413289186Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:44568"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:12.421977083Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"ee7b1448-b3bb-46f4-a11e-03d25675c837","remote_addr":"[fd00:1122:3344:101::1]:44568","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8616,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:12.424391361Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:11:12.424395391Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:16.325302692Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:16.325415063Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:61516"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:16.333895157Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"335557ee-c7ff-442b-9bab-411e2da280ab","remote_addr":"[fd00:1122:3344:101::1]:61516","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8338,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:16.336575Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:11:16.33658239Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:19.69598134Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:19.696176853Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:63610"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:19.704303161Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"f3d96d26-25c7-47f6-80b4-736c7302a91c","remote_addr":"[fd00:1122:3344:101::1]:63610","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8031,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:19.706542187Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:11:19.706546557Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:19.71815808Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"329491034640178752"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:19.718277492Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17995928283259856397"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:11:19.784354027Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"995.656341ms"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:20.655610431Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5398698369582487023"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:20.821023236Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15858676633825570490"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:20.821138588Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2459447259745793341"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:21.292342627Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:45019"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:21.292408739Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"93e68275-0733-460a-a318-85888f4b44d9","remote_addr":"[fdb0:200::2]:45019","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":24,"response_code":"200"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:22.313600263Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:22.313736085Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:28.803776686Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:28.803973559Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:41994"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:28.813109684Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"dd6585d7-e31b-45b2-8025-d17becfc0cb1","remote_addr":"[fd00:1122:3344:101::1]:41994","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9071,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:28.815633544Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:11:28.815640964Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:29.719992681Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"329491034640178752"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:29.720114023Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17995928283259856397"}
{"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:30.657855747Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"5398698369582487023"}
{"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:30.822690043Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"15858676633825570490"}
{"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:30.822912757Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"2459447259745793341"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:32.315219219Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:32.315326201Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:34.792848058Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:34.793039101Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:50500"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:34.801975713Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"e4497ca4-3934-4838-85c3-9f6073e7b027","remote_addr":"[fd00:1122:3344:101::1]:50500","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8838,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:34.804424472Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:11:34.804430282Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:11:35.815617678Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"1.759790623s"}
{"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:39.721612319Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"329491034640178752"}
{"msg":"expiring kstat after too many failures","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:39.721887674Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:485","error":"NoSuchKstat","reason":"Duration(60s)","id":"17995928283259856397"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:42.316839327Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:42.316952509Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:48.971222558Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:48.971406761Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:42642"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:48.981203296Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"8539606d-73c7-4e74-8f61-1a8b6809b4de","remote_addr":"[fd00:1122:3344:101::1]:42642","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9706,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:48.983607923Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:11:48.983613704Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:51.328101648Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:48480"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:51.328179699Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"873e47a6-8f54-4b8b-9462-d0241445c501","remote_addr":"[fdb0:200::2]:48480","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":33,"response_code":"200"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:52.319081425Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:11:52.319192227Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:11:52.612513514Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"5.732934179s"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:58.242698373Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:58.242847635Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:36808"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:58.251248988Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"ea7fcac9-cc22-4540-bdde-31d0b847bffa","remote_addr":"[fd00:1122:3344:101::1]:36808","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8340,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:11:58.253490913Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:11:58.253497193Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:12:02.32050209Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:12:02.320646363Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:12:12.322690388Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:12:12.32280362Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:12:13.380144716Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"10.839542407s"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:12:16.745139274Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:12:16.745328007Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:46943"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:12:16.754091986Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"4e3e9de3-3581-42c1-8f7c-2866d88ce291","remote_addr":"[fd00:1122:3344:101::1]:46943","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8682,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:12:16.756582085Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:12:16.756588765Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:12:21.362605192Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:65429"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:12:21.362682564Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"300fccf3-dcee-4956-932c-046f55911921","remote_addr":"[fdb0:200::2]:65429","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":31,"response_code":"200"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:12:22.325551846Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:12:22.325695098Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:12:32.327872364Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:12:32.327993306Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:12:34.218419063Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:12:34.218616846Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:60318"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:12:34.227521617Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"a696ced5-7019-4812-83c2-97d5ccf2f18d","remote_addr":"[fd00:1122:3344:101::1]:60318","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8833,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:12:34.231446359Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:12:34.231455089Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:12:39.256028363Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"10.025948118s"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:12:42.329941978Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:12:42.3300443Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:12:51.395708943Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:63229"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:12:51.395804234Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"7a602bb5-caaf-4f9d-a7ec-370eb0f4dcf9","remote_addr":"[fdb0:200::2]:63229","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":36,"response_code":"200"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:12:52.331541017Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:12:52.331647009Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:12:58.675412141Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:12:58.675574933Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:60367"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:12:58.684307181Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"29c5efcf-003d-4d11-9e09-b69f05f869ce","remote_addr":"[fd00:1122:3344:101::1]:60367","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8611,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:12:58.686906893Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:12:58.686914662Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:13:02.333760206Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:13:02.333864798Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:13:04.318032697Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"33.610211075s"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:13:12.335798922Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:13:12.335907113Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:13:13.100741404Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:13:13.100915827Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:57488"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:13:13.109776327Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"e340bfbb-00a1-47fe-89dd-25140671e964","remote_addr":"[fd00:1122:3344:101::1]:57488","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8759,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:13:13.11314719Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:13:13.113155931Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:13:21.428988818Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:33084"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:13:21.42909397Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"7b4c5c38-9b7d-4989-b8dc-43a296f08151","remote_addr":"[fdb0:200::2]:33084","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":36,"response_code":"200"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:13:22.33810664Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:13:22.338270863Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:13:32.340397189Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:13:32.340495641Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:13:35.217913861Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:13:35.218020272Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:62103"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:13:35.226645609Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"62627fdf-bf02-4d93-b284-d274046c053a","remote_addr":"[fd00:1122:3344:101::1]:62103","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8433,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:13:35.228981546Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:13:35.228987736Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:13:42.343270588Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:13:42.343530562Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:13:51.460856679Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:41757"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:13:51.460956531Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"82b1c7e4-535c-4e14-9da4-b24dfca6378b","remote_addr":"[fdb0:200::2]:41757","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":43,"response_code":"200"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:13:52.344904916Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:13:52.345061668Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:13:52.963624637Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"61.106793434s"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:14:02.346522965Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:14:02.346636907Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:14:12.348812904Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:14:12.348927856Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:14:14.097045542Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:14:14.097161724Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:59970"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:14:14.105683149Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"69a34c80-554b-430a-9073-ef7f981125b3","remote_addr":"[fd00:1122:3344:101::1]:59970","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8318,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:14:14.109496809Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:14:14.109507989Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:14:21.495014199Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:40075"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:14:21.49508217Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"9f05c37c-5f12-4db0-bd49-6d9c09c9bf5d","remote_addr":"[fdb0:200::2]:40075","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":27,"response_code":"200"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:14:22.350748927Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:14:22.350856739Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:14:32.353503663Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:14:32.353747237Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:14:42.355946465Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:14:42.356064017Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:14:47.85803655Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:14:47.858221123Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:55253"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:14:47.866545865Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"3c7634c8-05bf-4613-9710-f2c8a798ff32","remote_addr":"[fd00:1122:3344:101::1]:55253","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8229,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:14:47.870422326Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:14:47.870435396Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:14:51.52861882Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:37568"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:14:51.528673291Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"08fa8e36-d002-4730-b4f2-4e0545b86236","remote_addr":"[fdb0:200::2]:37568","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":26,"response_code":"200"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:14:52.358243905Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:14:52.358356567Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:15:02.359860824Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:15:02.359965456Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:15:09.107672172Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"187.159996811s"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:15:12.361503753Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:15:12.361712796Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:15:15.251367129Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:15:21.563579672Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:50806"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:15:21.563672113Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"3ebcaf5f-640e-42f5-b9c1-512308b8a3a3","remote_addr":"[fdb0:200::2]:50806","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":36,"response_code":"200"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:15:22.363990326Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:15:22.364100108Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:15:29.021548916Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:15:29.021656858Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:45886"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:15:29.030717651Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"fcc61e0a-7afb-4e11-9176-a9806732299c","remote_addr":"[fd00:1122:3344:101::1]:45886","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8878,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:15:29.033300112Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:15:29.033315492Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:15:32.365687215Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:15:32.365807477Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:15:42.367819452Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:15:42.367934834Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:15:51.597683439Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:42923"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:15:51.59775852Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"c424417d-28eb-46fa-8003-30ced83f7b93","remote_addr":"[fdb0:200::2]:42923","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":27,"response_code":"200"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:15:52.369611362Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:15:52.369741124Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:16:02.371527876Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:16:02.371662098Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:16:12.373834084Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:16:12.373955376Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:16:21.631886399Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:53615"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:16:21.63195204Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"f73103ef-0d3b-4e66-a813-48f1ea381ae0","remote_addr":"[fdb0:200::2]:53615","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":28,"response_code":"200"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:16:22.376639483Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:16:22.376854027Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:16:32.378494973Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:16:32.378638676Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:16:42.380544018Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:16:42.38066024Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:16:43.82974401Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:16:43.829859942Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:36021"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:16:43.838578329Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"096bf963-10c4-47c8-ab92-2334640d99be","remote_addr":"[fd00:1122:3344:101::1]:36021","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8523,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:16:43.841078259Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:16:43.841083779Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:16:51.667350395Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:40546"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:16:51.667474327Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"979ff9e5-bafb-42af-b346-41b49c511c28","remote_addr":"[fdb0:200::2]:40546","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":45,"response_code":"200"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:16:52.382921668Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:16:52.383026609Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:17:02.38552829Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:17:02.385652322Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:17:12.387116337Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:17:12.387223558Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:17:21.701589711Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:41875"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:17:21.701666113Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"e8b640b3-3f14-480a-be27-fe4798721d43","remote_addr":"[fdb0:200::2]:41875","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":37,"response_code":"200"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:17:22.389484006Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:17:22.389596958Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:17:27.761602086Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:17:27.761763648Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:59235"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:17:27.770371174Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"a2dea341-7057-4a73-a181-f5b02e62b278","remote_addr":"[fd00:1122:3344:101::1]:59235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8535,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:17:27.773041916Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:17:27.773050916Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:17:32.391689733Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:17:32.391790835Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:17:42.393889609Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:17:42.394009221Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:17:51.737193829Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:52366"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:17:51.737304211Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"e89b4897-a644-4967-bf99-d9669191c408","remote_addr":"[fdb0:200::2]:52366","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":46,"response_code":"200"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:17:52.395791821Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:17:52.396054335Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:18:02.398426065Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:18:02.398526076Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:18:12.401183471Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:18:12.401427115Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:18:21.769385935Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:61779"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:18:21.769433146Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"4980c812-e597-4fc1-a833-5ea20bb84d16","remote_addr":"[fdb0:200::2]:61779","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":28,"response_code":"200"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:18:22.403033353Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:18:22.403145735Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:18:31.30339008Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"177.567589341s"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:18:32.404939835Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:18:32.405049767Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:18:42.40689745Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:18:42.407134423Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:18:51.803895928Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:57586"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:18:51.803955849Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"09d681ce-839c-44bc-b2d1-2ef71f847cbc","remote_addr":"[fdb0:200::2]:57586","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":25,"response_code":"200"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:18:52.408470755Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:18:52.408586267Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:18:59.446175735Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:18:59.446358138Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:38614"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_71e73077-bc8e-4f9f-8689-878cda8ba43e': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:18:59.455159248Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3958"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:18:59.45530912Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"22c729bf-bb47-4833-a435-4018979f9e51","remote_addr":"[fd00:1122:3344:101::1]:38614","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8848,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-21T23:18:59.459356034Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:19:02.411056807Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:19:02.411166479Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:19:12.412829167Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:19:12.41298325Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:19:21.837542796Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:200::2]:50876"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-21T23:19:21.837614417Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"65db33d1-c795-4f8c-a695-f90e490d770b","remote_addr":"[fdb0:200::2]:50876","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":34,"response_code":"200"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:19:22.415188248Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"9374031121665754717"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-21T23:19:22.415453232Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"df02b17d-a4bd-4ac6-9447-5abe2628a534","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12244715441543845500"}