[ Jul 18 20:38:21 Enabled. ] [ Jul 18 20:38:21 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Jul 18 20:38:21 Method "start" exited with status 0. ] [ Jul 18 20:38:21 Rereading configuration. ] [ Jul 18 20:38:21 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:21.067934632Z","hostname":"buskin","pid":1555,"file":"illumos-utils/src/running_zone.rs:110"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:21.068101305Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:21.122135225Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:21.122182906Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:21.240017914Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:21.248551164Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:21.256592106Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:21.859608148Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:9fb2::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.005865928Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:125"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.005904039Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:137"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.00598913Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:150"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.006089402Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:172","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.006115572Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.006134652Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:718"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.010223284Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:625"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:22.010235405Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:767"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.010262455Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:186"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.010271725Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:826"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.010296905Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.010370667Z","hostname":"buskin","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.010503509Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.010531729Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.010545599Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.023836691Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.023871481Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:372","id":"9376dea5-4006-4708-b1ab-6628a9deda68 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.122274075Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 9376dea5-4006-4708-b1ab-6628a9deda68 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.333335529Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.333376149Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.333382479Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.333393579Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.333402299Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.33342172Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.33342759Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.342726931Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.342752212Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:372","id":"d973a939-4d3f-45e3-a19e-2f06e6f1fe72 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.431457438Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: d973a939-4d3f-45e3-a19e-2f06e6f1fe72 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639205971Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639259082Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 9376dea5-4006-4708-b1ab-6628a9deda68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d973a939-4d3f-45e3-a19e-2f06e6f1fe72 (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639298542Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639305633Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639310973Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639317973Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639325053Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639330163Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 9376dea5-4006-4708-b1ab-6628a9deda68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d973a939-4d3f-45e3-a19e-2f06e6f1fe72 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639345123Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639350443Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639355123Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639371803Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639377094Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639381154Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 9376dea5-4006-4708-b1ab-6628a9deda68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d973a939-4d3f-45e3-a19e-2f06e6f1fe72 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639398344Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639403294Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639407994Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639413334Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639418434Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_3.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639430744Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639436764Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639455765Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 9376dea5-4006-4708-b1ab-6628a9deda68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d973a939-4d3f-45e3-a19e-2f06e6f1fe72 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639483685Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_4.vdev"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639490035Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639495916Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639500596Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639505945Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639511466Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 9376dea5-4006-4708-b1ab-6628a9deda68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d973a939-4d3f-45e3-a19e-2f06e6f1fe72 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639531006Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_5.vdev"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639535446Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639541226Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639545896Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639550106Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639557166Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 9376dea5-4006-4708-b1ab-6628a9deda68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d973a939-4d3f-45e3-a19e-2f06e6f1fe72 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639582697Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639593217Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639597407Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639602217Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639607217Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639611827Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 9376dea5-4006-4708-b1ab-6628a9deda68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d973a939-4d3f-45e3-a19e-2f06e6f1fe72 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639641048Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639647208Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639651088Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639655818Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639661608Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639666238Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 9376dea5-4006-4708-b1ab-6628a9deda68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d973a939-4d3f-45e3-a19e-2f06e6f1fe72 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639702748Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639708289Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639712479Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639717739Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639722259Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639727149Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 9376dea5-4006-4708-b1ab-6628a9deda68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d973a939-4d3f-45e3-a19e-2f06e6f1fe72 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639761489Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639766849Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.63977136Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:96"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.63978039Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:98"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.63978641Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 9376dea5-4006-4708-b1ab-6628a9deda68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d973a939-4d3f-45e3-a19e-2f06e6f1fe72 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.6398119Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.63981573Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.63982017Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.63982353Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:210"} {"msg":"No ledger in /pool/int/9376dea5-4006-4708-b1ab-6628a9deda68/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639827101Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d973a939-4d3f-45e3-a19e-2f06e6f1fe72/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.63983637Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639844771Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/9376dea5-4006-4708-b1ab-6628a9deda68/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639854141Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d973a939-4d3f-45e3-a19e-2f06e6f1fe72/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639860921Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639867491Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639896531Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:215"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.639914912Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:230"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.65099471Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:722"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.651037641Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3583"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.651069181Z","hostname":"buskin","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:292","time_to_next_cleanup":"599.999912298s","next_cleanup":"Instant { tv_sec: 1558, tv_nsec: 926891500 }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.651085341Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No ledger in /pool/int/9376dea5-4006-4708-b1ab-6628a9deda68/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.651147152Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d973a939-4d3f-45e3-a19e-2f06e6f1fe72/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.653090212Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.654102717Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:205"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.654212289Z","hostname":"buskin","pid":1555,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.654441262Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1011"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.654449332Z","hostname":"buskin","pid":1555,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.654488723Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No ledger in /pool/int/9376dea5-4006-4708-b1ab-6628a9deda68/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.654538054Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d973a939-4d3f-45e3-a19e-2f06e6f1fe72/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.654551934Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.654556784Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1071"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.654577604Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No ledger in /pool/int/9376dea5-4006-4708-b1ab-6628a9deda68/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.654587685Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d973a939-4d3f-45e3-a19e-2f06e6f1fe72/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.654596505Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.654601785Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1112"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.654605475Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:156"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.654609795Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:129"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.654631625Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Writing ledger to /pool/int/9376dea5-4006-4708-b1ab-6628a9deda68/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.654671246Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d973a939-4d3f-45e3-a19e-2f06e6f1fe72/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.654999821Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.655415797Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:176"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.655429647Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1145"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.655511308Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/9376dea5-4006-4708-b1ab-6628a9deda68/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.655520429Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d973a939-4d3f-45e3-a19e-2f06e6f1fe72/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.655678771Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.655798823Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.655853544Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: d7d0bedb-61a2-44cf-b5a5-637dc79ea041, rack_id: fd37384b-14c5-4709-b9c5-1d347e1fa35d, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.656077167Z","hostname":"buskin","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:53906","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: d7d0bedb-61a2-44cf-b5a5-637dc79ea041, rack_id: fd37384b-14c5-4709-b9c5-1d347e1fa35d, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.656171218Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:360"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.656179819Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:375"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.656208459Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.65623313Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/server.rs:44"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.65623695Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:572"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.656240219Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.65624563Z","hostname":"buskin","pid":1555,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"No ledger in /pool/int/9376dea5-4006-4708-b1ab-6628a9deda68/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.65626162Z","hostname":"buskin","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d973a939-4d3f-45e3-a19e-2f06e6f1fe72/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.6562753Z","hostname":"buskin","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.65627955Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:561"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.65628615Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:589"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.65628936Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:393"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.656321651Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:374"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.656334731Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:479","result":"DisksManagementResult { status: [] }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.658203489Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3815"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.68788682Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.707915764Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.725127575Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.735437002Z","hostname":"buskin","pid":1555,"sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:366"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.735493142Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.735497833Z","hostname":"buskin","pid":1555,"sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:378"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.735573034Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 70.350198ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.73864525Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c730b8b3-93bd-4775-9d2d-03754beddbbe\", \"content-length\": \"124\", \"date\": \"Thu, 18 Jul 2024 20:38:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c730b8b3-93bd-4775-9d2d-03754beddbbe\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_9376dea5-4006-4708-b1ab-6628a9deda68/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.749876551Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_9376dea5-4006-4708-b1ab-6628a9deda68/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.772726718Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_9376dea5-4006-4708-b1ab-6628a9deda68/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.772772238Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.778373313Z","hostname":"buskin","pid":1555,"sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:393"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.778385684Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_9376dea5-4006-4708-b1ab-6628a9deda68/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.778390784Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.811124521Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 68.873973ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.811851861Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4218a75c-d2b2-4791-9607-fb5c77b8fb8e\", \"content-length\": \"124\", \"date\": \"Thu, 18 Jul 2024 20:38:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4218a75c-d2b2-4791-9607-fb5c77b8fb8e\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.854984406Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.881874074Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 98.17941ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.882404402Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4eadfa98-5d63-414b-a24f-da1a83c6d2aa\", \"content-length\": \"124\", \"date\": \"Thu, 18 Jul 2024 20:38:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4eadfa98-5d63-414b-a24f-da1a83c6d2aa\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.981490446Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 105.1368ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:22.981915822Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c69a7805-7774-40dc-b57b-f595c2a4bf4a\", \"content-length\": \"124\", \"date\": \"Thu, 18 Jul 2024 20:38:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c69a7805-7774-40dc-b57b-f595c2a4bf4a\" }"} {"msg":"Mounting oxi_9376dea5-4006-4708-b1ab-6628a9deda68/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.0746591Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.08846651Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.088517371Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.088554211Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.088591852Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.088607582Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Failed enable ddm stats (will retry after 278.902522ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.091070469Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3be2acf0-b1fb-4f05-95eb-9c0b9e755bf6\", \"content-length\": \"124\", \"date\": \"Thu, 18 Jul 2024 20:38:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3be2acf0-b1fb-4f05-95eb-9c0b9e755bf6\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.371520706Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 348.905571ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.372477401Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d36e5db8-19db-435d-b42e-7dd3020916a8\", \"content-length\": \"124\", \"date\": \"Thu, 18 Jul 2024 20:38:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d36e5db8-19db-435d-b42e-7dd3020916a8\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.722626625Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 313.627113ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.723306046Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"67fffc13-a981-456b-ae5e-75c7e2ca874e\", \"content-length\": \"124\", \"date\": \"Thu, 18 Jul 2024 20:38:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"67fffc13-a981-456b-ae5e-75c7e2ca874e\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.812782544Z","hostname":"buskin","pid":1555,"sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.813380703Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:33450","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:205","component":"producer-server","component":"MetricsManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:205"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.813406853Z","hostname":"buskin","pid":1555,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.813483094Z","hostname":"buskin","pid":1555,"file":"oximeter/producer/src/lib.rs:283","component":"producer-server","component":"MetricsManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"oximeter/producer/src/lib.rs:283","interval":"30s","address":"[fd00:1122:3344:101::1]:33450","producer_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:23.814734883Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"282.75283ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.858753931Z","hostname":"buskin","pid":1555,"component":"VmmReservoirManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.858793102Z","hostname":"buskin","pid":1555,"component":"VmmReservoirManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.858921974Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:905","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.858929594Z","hostname":"buskin","pid":1555,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.859736006Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:205"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.859778687Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:430"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.859806907Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Writing ledger to /pool/int/9376dea5-4006-4708-b1ab-6628a9deda68/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.859833198Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d973a939-4d3f-45e3-a19e-2f06e6f1fe72/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.860053911Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.860212044Z","hostname":"buskin","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.860273554Z","hostname":"buskin","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.1.100, vlan_id: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.860296395Z","hostname":"buskin","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:53906","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.860306105Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3583"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.860350235Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:23.860357256Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.055365315Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No ledger in /pool/int/9376dea5-4006-4708-b1ab-6628a9deda68/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.055448587Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d973a939-4d3f-45e3-a19e-2f06e6f1fe72/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.055473337Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.055583429Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.055598469Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.055739281Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d7d0bedb-61a2-44cf-b5a5-637dc79ea041): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:24.060991581Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.087786918Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:63509"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.088021871Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.088164033Z","hostname":"buskin","pid":1555,"uri":"/inventory","method":"GET","req_id":"02e1e3f0-bcd1-4966-8182-a80c012da21a","remote_addr":"[fd00:1122:3344:101::1]:63509","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":238,"response_code":"200"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:24.099088529Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"346.224998ms"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.118307071Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:62950"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.118462513Z","hostname":"buskin","pid":1555,"uri":"/sled-role","method":"GET","req_id":"09195dc9-80fa-4c98-9a8d-5969e25f3895","remote_addr":"[fd00:1122:3344:101::1]:62950","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":67,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.120007127Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Writing ledger to /pool/int/9376dea5-4006-4708-b1ab-6628a9deda68/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.120043787Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d973a939-4d3f-45e3-a19e-2f06e6f1fe72/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.120468094Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.120727078Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:832"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: 158b1caa-2228-499a-bdc5-f4c80dc90af1, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, pool_id: a7295e5f-277c-4ceb-8d2f-53dc611d3d65 (zpool) }, OmicronPhysicalDiskConfig { id: 1e4bb762-00e4-4611-bf4a-e132988c124d, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, pool_id: 5fc83c59-f3a1-4ff2-a7e1-d5460eabb2b9 (zpool) }, OmicronPhysicalDiskConfig { id: d066bb9b-e538-42b5-ae78-4baa380d62ba, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, pool_id: c8c7ad3c-bcaf-447c-ab38-5a542c9f0928 (zpool) }, OmicronPhysicalDiskConfig { id: 8efb5881-50b6-41a7-8031-ff6e577da3aa, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, pool_id: 61b850b7-5d2e-4dcf-b938-603fca674806 (zpool) }, OmicronPhysicalDiskConfig { id: 66c05227-c95b-409e-ab43-abffbb492a1e, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, pool_id: 041bb4d1-3950-4b3e-8f1e-68d1fe20601c (zpool) }, OmicronPhysicalDiskConfig { id: 77d9b5b7-31b6-4ed4-877a-b7f758c1a5f0, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, pool_id: 54795d16-5339-4df9-9c09-fc1cf6ac673f (zpool) }, OmicronPhysicalDiskConfig { id: f4c7db9d-0867-4994-b101-aa6333c4c5d5, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, pool_id: cc5bc32f-a391-4b88-a146-484bf21b4600 (zpool) }, OmicronPhysicalDiskConfig { id: bf0a4622-3acc-432c-93b5-2ae7eef915e7, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, pool_id: 6c7b4f4d-f40e-46f2-97a2-82e7a4b14ce6 (zpool) }, OmicronPhysicalDiskConfig { id: c6da3599-93bd-4957-9c52-29f29792cfec, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, pool_id: dd0a5b6d-3c34-45ab-9063-5a5904f2e1ae (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.152081383Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:348"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.152179155Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:54421"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.152390368Z","hostname":"buskin","pid":1555,"sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:825"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 158b1caa-2228-499a-bdc5-f4c80dc90af1, pool_id: a7295e5f-277c-4ceb-8d2f-53dc611d3d65 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 1e4bb762-00e4-4611-bf4a-e132988c124d, pool_id: 5fc83c59-f3a1-4ff2-a7e1-d5460eabb2b9 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: d066bb9b-e538-42b5-ae78-4baa380d62ba, pool_id: c8c7ad3c-bcaf-447c-ab38-5a542c9f0928 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 8efb5881-50b6-41a7-8031-ff6e577da3aa, pool_id: 61b850b7-5d2e-4dcf-b938-603fca674806 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 66c05227-c95b-409e-ab43-abffbb492a1e, pool_id: 041bb4d1-3950-4b3e-8f1e-68d1fe20601c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 77d9b5b7-31b6-4ed4-877a-b7f758c1a5f0, pool_id: 54795d16-5339-4df9-9c09-fc1cf6ac673f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: f4c7db9d-0867-4994-b101-aa6333c4c5d5, pool_id: cc5bc32f-a391-4b88-a146-484bf21b4600 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: bf0a4622-3acc-432c-93b5-2ae7eef915e7, pool_id: 6c7b4f4d-f40e-46f2-97a2-82e7a4b14ce6 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: c6da3599-93bd-4957-9c52-29f29792cfec, pool_id: dd0a5b6d-3c34-45ab-9063-5a5904f2e1ae (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.152434859Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No ledger in /pool/int/9376dea5-4006-4708-b1ab-6628a9deda68/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.152449679Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d973a939-4d3f-45e3-a19e-2f06e6f1fe72/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.152459759Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.152471739Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:643"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.15251909Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:374"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.1525275Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.15253578Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.15254244Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.243288398Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:24.243329908Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"a7295e5f-277c-4ceb-8d2f-53dc611d3d65 (zpool)"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:24.597329692Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"658.830902ms"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d7d0bedb-61a2-44cf-b5a5-637dc79ea041): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-18T20:38:24.861899597Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:25.257541542Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: a7295e5f-277c-4ceb-8d2f-53dc611d3d65 (zpool), kind: External }"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:25.417622592Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.693429443s"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:25.423464751Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:25.423564922Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:25.423576052Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:25.423589882Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:25.423613753Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:25.423750725Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:25.423759495Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:25.967589439Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:26.156407304Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d7d0bedb-61a2-44cf-b5a5-637dc79ea041): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:26.580904747Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d7d0bedb-61a2-44cf-b5a5-637dc79ea041): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-18T20:38:26.580971458Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Automatically destroying dataset: oxp_a7295e5f-277c-4ceb-8d2f-53dc611d3d65/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:26.58111069Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 74.05341ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:26.758889069Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 107.975668ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:27.305547476Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:27.307465735Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.080747947s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 103.844501ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:27.936872328Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d7d0bedb-61a2-44cf-b5a5-637dc79ea041): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-18T20:38:28.113300056Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 410.10873ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:28.272656184Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d7d0bedb-61a2-44cf-b5a5-637dc79ea041): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:28.438251938Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 415.082093ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:29.30879386Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d7d0bedb-61a2-44cf-b5a5-637dc79ea041): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-18T20:38:29.311673504Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Looking for unencrypted datasets in oxp_a7295e5f-277c-4ceb-8d2f-53dc611d3d65","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:29.322891394Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:29.773668696Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"8.94240887s"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:29.774244175Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:29.774255225Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:29.774285595Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:29.774291386Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:29.774295236Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 568.07675ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:29.991177587Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:30.000391577Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:30.000401217Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"5fc83c59-f3a1-4ff2-a7e1-d5460eabb2b9 (zpool)"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d7d0bedb-61a2-44cf-b5a5-637dc79ea041): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:30.709294217Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d7d0bedb-61a2-44cf-b5a5-637dc79ea041): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-18T20:38:30.709447359Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 799.608104ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:30.935969447Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:31.486239019Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 5fc83c59-f3a1-4ff2-a7e1-d5460eabb2b9 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:32.007356659Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:32.007408509Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:32.00741684Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:32.00743968Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:32.007484831Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:32.007645783Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:32.007655553Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d7d0bedb-61a2-44cf-b5a5-637dc79ea041): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-18T20:38:33.249157637Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d7d0bedb-61a2-44cf-b5a5-637dc79ea041): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:33.249433631Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:33.478905444Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_5fc83c59-f3a1-4ff2-a7e1-d5460eabb2b9/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:33.959340466Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.183599389Z","hostname":"buskin","pid":1555,"component":"VmmReservoirManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d7d0bedb-61a2-44cf-b5a5-637dc79ea041): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-18T20:38:34.253745744Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d7d0bedb-61a2-44cf-b5a5-637dc79ea041): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:34.281287392Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Looking for unencrypted datasets in oxp_5fc83c59-f3a1-4ff2-a7e1-d5460eabb2b9","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.304991932Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.313333658Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.313379349Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.313388239Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.313410619Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.313418539Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.322965214Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.323001925Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"c8c7ad3c-bcaf-447c-ab38-5a542c9f0928 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.418536385Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: c8c7ad3c-bcaf-447c-ab38-5a542c9f0928 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.426647518Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.426714089Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.426722989Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.42677732Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.42678786Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.426984823Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.426992203Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.463198983Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_c8c7ad3c-bcaf-447c-ab38-5a542c9f0928/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.470633936Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_c8c7ad3c-bcaf-447c-ab38-5a542c9f0928","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.593166795Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.600857952Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.600908453Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.600916603Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.600924033Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.600930513Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.610553329Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.6105894Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"61b850b7-5d2e-4dcf-b938-603fca674806 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.705923517Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 61b850b7-5d2e-4dcf-b938-603fca674806 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.713666814Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.713708015Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.713714735Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.713751096Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.713762846Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.713926858Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.713935839Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.751650921Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_61b850b7-5d2e-4dcf-b938-603fca674806/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.759702223Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_61b850b7-5d2e-4dcf-b938-603fca674806","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.884297974Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.892362967Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.892400137Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.892407617Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.892412957Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.892418857Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.902268857Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.902299027Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"041bb4d1-3950-4b3e-8f1e-68d1fe20601c (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.991327178Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 041bb4d1-3950-4b3e-8f1e-68d1fe20601c (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.998327075Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.998406506Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.998417246Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.998453647Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.998464277Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.998597359Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:34.998605549Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.033905515Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_041bb4d1-3950-4b3e-8f1e-68d1fe20601c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.041578931Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_041bb4d1-3950-4b3e-8f1e-68d1fe20601c","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.157773405Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.164822942Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.164872092Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.164895813Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.164903543Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.164916553Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.174614411Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.174647621Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"54795d16-5339-4df9-9c09-fc1cf6ac673f (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.266132Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: 54795d16-5339-4df9-9c09-fc1cf6ac673f (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.274127901Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.274172702Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.274182622Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.274229302Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.274243352Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.274455206Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.274472356Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d7d0bedb-61a2-44cf-b5a5-637dc79ea041): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-18T20:38:35.278259284Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.313468788Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_54795d16-5339-4df9-9c09-fc1cf6ac673f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.322527405Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_54795d16-5339-4df9-9c09-fc1cf6ac673f","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.450171193Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.458112253Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.458168584Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.458175934Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.458184335Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.458190244Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.468626913Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.468662323Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"cc5bc32f-a391-4b88-a146-484bf21b4600 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.565393662Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: cc5bc32f-a391-4b88-a146-484bf21b4600 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.573617187Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.573660667Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.573666597Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.573713578Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.573722178Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.573916441Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.573925721Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.611923168Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_cc5bc32f-a391-4b88-a146-484bf21b4600/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.620252824Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.638268938Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2256"} {"msg":"Looking for unencrypted datasets in oxp_cc5bc32f-a391-4b88-a146-484bf21b4600","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.750833166Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.758508023Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.758552303Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.758571354Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.758576594Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.758589654Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.76819954Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.76823109Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"6c7b4f4d-f40e-46f2-97a2-82e7a4b14ce6 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.863876642Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 6c7b4f4d-f40e-46f2-97a2-82e7a4b14ce6 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.871980605Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.872017776Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.872025506Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.872048316Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.872059066Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.872208168Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.872217289Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.90722917Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_6c7b4f4d-f40e-46f2-97a2-82e7a4b14ce6/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:35.915207891Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_6c7b4f4d-f40e-46f2-97a2-82e7a4b14ce6","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.039970775Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.048251231Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.048286331Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.048294051Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.048299021Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.048304861Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.058149381Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.058182961Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"dd0a5b6d-3c34-45ab-9063-5a5904f2e1ae (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.155307345Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: dd0a5b6d-3c34-45ab-9063-5a5904f2e1ae (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.16354023Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.163635662Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.163654722Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.163670962Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.163683723Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.163881706Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.163892156Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.20175929Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_dd0a5b6d-3c34-45ab-9063-5a5904f2e1ae/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.209963115Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d7d0bedb-61a2-44cf-b5a5-637dc79ea041): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-18T20:38:36.282552007Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d7d0bedb-61a2-44cf-b5a5-637dc79ea041): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:36.282821021Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Looking for unencrypted datasets in oxp_dd0a5b6d-3c34-45ab-9063-5a5904f2e1ae","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.337845856Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.347422531Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.347461942Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/9376dea5-4006-4708-b1ab-6628a9deda68/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.347504322Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.347624204Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 9376dea5-4006-4708-b1ab-6628a9deda68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d973a939-4d3f-45e3-a19e-2f06e6f1fe72 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: a7295e5f-277c-4ceb-8d2f-53dc611d3d65 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 5fc83c59-f3a1-4ff2-a7e1-d5460eabb2b9 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: c8c7ad3c-bcaf-447c-ab38-5a542c9f0928 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 61b850b7-5d2e-4dcf-b938-603fca674806 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 041bb4d1-3950-4b3e-8f1e-68d1fe20601c (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 54795d16-5339-4df9-9c09-fc1cf6ac673f (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: cc5bc32f-a391-4b88-a146-484bf21b4600 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6c7b4f4d-f40e-46f2-97a2-82e7a4b14ce6 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: dd0a5b6d-3c34-45ab-9063-5a5904f2e1ae (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.347667595Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.347677955Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/d973a939-4d3f-45e3-a19e-2f06e6f1fe72/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.347807397Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.34803335Z","hostname":"buskin","pid":1555,"sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:829"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.348041721Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.348046231Z","hostname":"buskin","pid":1555,"sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:855","generation":"Generation(1)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.348051731Z","hostname":"buskin","pid":1555,"sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:860"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.348057531Z","hostname":"buskin","pid":1555,"sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:865"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.348063811Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:116","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.348070511Z","hostname":"buskin","pid":1555,"sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:871"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.348114592Z","hostname":"buskin","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:822","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.348122262Z","hostname":"buskin","pid":1555,"sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:876"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.348204893Z","hostname":"buskin","pid":1555,"uri":"/omicron-physical-disks","method":"PUT","req_id":"bae766fd-089e-4ce7-b008-997cf24aecaa","remote_addr":"[fd00:1122:3344:101::1]:54421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":12195979,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: ed87b889-28ea-4b87-95a0-218df6659a2b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a7295e5f-277c-4ceb-8d2f-53dc611d3d65 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7295e5f-277c-4ceb-8d2f-53dc611d3d65 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cec09772-8550-421d-a72a-2ed147e3a7eb, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5fc83c59-f3a1-4ff2-a7e1-d5460eabb2b9 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5fc83c59-f3a1-4ff2-a7e1-d5460eabb2b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 4201b309-328f-4a43-9074-978b08f72f31, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: c8c7ad3c-bcaf-447c-ab38-5a542c9f0928 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8c7ad3c-bcaf-447c-ab38-5a542c9f0928 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.38030249Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.380443202Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:46362"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed87b889-28ea-4b87-95a0-218df6659a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: a7295e5f-277c-4ceb-8d2f-53dc611d3d65 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.380715097Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.380721317Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ed87b889-28ea-4b87-95a0-218df6659a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: a7295e5f-277c-4ceb-8d2f-53dc611d3d65 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cec09772-8550-421d-a72a-2ed147e3a7eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5fc83c59-f3a1-4ff2-a7e1-d5460eabb2b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.431412336Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.431456797Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: cec09772-8550-421d-a72a-2ed147e3a7eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5fc83c59-f3a1-4ff2-a7e1-d5460eabb2b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4201b309-328f-4a43-9074-978b08f72f31, dataset_name: DatasetName { pool_name: ZpoolName { id: c8c7ad3c-bcaf-447c-ab38-5a542c9f0928 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.480918527Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.480960488Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4201b309-328f-4a43-9074-978b08f72f31, dataset_name: DatasetName { pool_name: ZpoolName { id: c8c7ad3c-bcaf-447c-ab38-5a542c9f0928 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.530165575Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No ledger in /pool/int/9376dea5-4006-4708-b1ab-6628a9deda68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.530208716Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d973a939-4d3f-45e3-a19e-2f06e6f1fe72/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.530224666Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.530284997Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.574402377Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.574453117Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.574459187Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_ed87b889-28ea-4b87-95a0-218df6659a2b","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.626170942Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_4201b309-328f-4a43-9074-978b08f72f31","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.629792897Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_cec09772-8550-421d-a72a-2ed147e3a7eb","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.633793578Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_internal_dns_ed87b889-28ea-4b87-95a0-218df6659a2b","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.646674333Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_4201b309-328f-4a43-9074-978b08f72f31","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.650940228Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_cec09772-8550-421d-a72a-2ed147e3a7eb","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:36.654501932Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d7d0bedb-61a2-44cf-b5a5-637dc79ea041): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-18T20:38:37.285225665Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Profile for oxz_internal_dns_4201b309-328f-4a43-9074-978b08f72f31:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:38.163732108Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:38.163909181Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:38.164218016Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_4201b309-328f-4a43-9074-978b08f72f31","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_internal_dns_cec09772-8550-421d-a72a-2ed147e3a7eb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:38.652750151Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:38.652827002Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_ed87b889-28ea-4b87-95a0-218df6659a2b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:39.132419281Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_internal_dns_4201b309-328f-4a43-9074-978b08f72f31\") failed: Property not found. retry in 54.249495ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:39.139909895Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_4201b309-328f-4a43-9074-978b08f72f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:39.139970536Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_cec09772-8550-421d-a72a-2ed147e3a7eb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:39.143930966Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:39.143992467Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_ed87b889-28ea-4b87-95a0-218df6659a2b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_internal_dns_4201b309-328f-4a43-9074-978b08f72f31\") failed: Property not found. retry in 53.859499ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:39.205075314Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_4201b309-328f-4a43-9074-978b08f72f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_cec09772-8550-421d-a72a-2ed147e3a7eb\") failed: Property not found. retry in 41.902475ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:39.261423529Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_cec09772-8550-421d-a72a-2ed147e3a7eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ed87b889-28ea-4b87-95a0-218df6659a2b\") failed: Property not found. retry in 53.686067ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:39.268275433Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_ed87b889-28ea-4b87-95a0-218df6659a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4201b309-328f-4a43-9074-978b08f72f31\") failed: Property not found. retry in 158.873363ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:39.275214369Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_4201b309-328f-4a43-9074-978b08f72f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_cec09772-8550-421d-a72a-2ed147e3a7eb\") failed: Property not found. retry in 102.676598ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:39.310799069Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_cec09772-8550-421d-a72a-2ed147e3a7eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ed87b889-28ea-4b87-95a0-218df6659a2b\") failed: Property not found. retry in 58.09734ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:39.398319507Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_ed87b889-28ea-4b87-95a0-218df6659a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_cec09772-8550-421d-a72a-2ed147e3a7eb\") failed: Property not found. retry in 271.360375ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:39.451846059Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_cec09772-8550-421d-a72a-2ed147e3a7eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4201b309-328f-4a43-9074-978b08f72f31\") failed: Property not found. retry in 231.503525ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:39.467307284Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_4201b309-328f-4a43-9074-978b08f72f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ed87b889-28ea-4b87-95a0-218df6659a2b\") failed: Property not found. retry in 167.817516ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:39.479014852Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_ed87b889-28ea-4b87-95a0-218df6659a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ed87b889-28ea-4b87-95a0-218df6659a2b\") failed: Property not found. retry in 499.589541ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:39.664769491Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_ed87b889-28ea-4b87-95a0-218df6659a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4201b309-328f-4a43-9074-978b08f72f31\") failed: Property not found. retry in 491.076159ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:39.712996073Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_4201b309-328f-4a43-9074-978b08f72f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_cec09772-8550-421d-a72a-2ed147e3a7eb\") failed: Property not found. retry in 327.22435ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:39.741968313Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_cec09772-8550-421d-a72a-2ed147e3a7eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_cec09772-8550-421d-a72a-2ed147e3a7eb\") failed: Property not found. retry in 1.127140451s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:40.079030259Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_cec09772-8550-421d-a72a-2ed147e3a7eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ed87b889-28ea-4b87-95a0-218df6659a2b\") failed: Property not found. retry in 497.162951ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:40.172745771Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_ed87b889-28ea-4b87-95a0-218df6659a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4201b309-328f-4a43-9074-978b08f72f31\") failed: Property not found. retry in 1.188486616s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:40.212365082Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_4201b309-328f-4a43-9074-978b08f72f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ed87b889-28ea-4b87-95a0-218df6659a2b\") failed: Property not found. retry in 865.847952ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:40.679436812Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_ed87b889-28ea-4b87-95a0-218df6659a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_cec09772-8550-421d-a72a-2ed147e3a7eb\") failed: Property not found. retry in 1.386824289s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:41.217264104Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_cec09772-8550-421d-a72a-2ed147e3a7eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4201b309-328f-4a43-9074-978b08f72f31\") failed: Property not found. retry in 935.908721ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:41.408763511Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_4201b309-328f-4a43-9074-978b08f72f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_ed87b889-28ea-4b87-95a0-218df6659a2b\") failed: Property not found. retry in 602.447664ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:41.552485242Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_ed87b889-28ea-4b87-95a0-218df6659a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:9fb2::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:41.693592354Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2273"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:41.693606824Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:578"} {"msg":"wait for service Some(\"oxz_internal_dns_ed87b889-28ea-4b87-95a0-218df6659a2b\") failed: Property not found. retry in 1.10642553s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:42.16441003Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_ed87b889-28ea-4b87-95a0-218df6659a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fed8:3788","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:42.235385327Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2280"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:42.240486675Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2341"} {"msg":"wait for service Some(\"oxz_internal_dns_4201b309-328f-4a43-9074-978b08f72f31\") failed: Property not found. retry in 1.152829159s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:42.352707558Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_4201b309-328f-4a43-9074-978b08f72f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:42.361140126Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2526"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:42.599057177Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2690"} {"msg":"wait for service Some(\"oxz_internal_dns_cec09772-8550-421d-a72a-2ed147e3a7eb\") failed: Property not found. retry in 1.318022251s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:42.612980958Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_cec09772-8550-421d-a72a-2ed147e3a7eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:42.788186388Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2405"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:43.091378309Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2440"} {"msg":"wait for service Some(\"oxz_internal_dns_ed87b889-28ea-4b87-95a0-218df6659a2b\") failed: Property not found. retry in 1.107340636s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:43.279406643Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_ed87b889-28ea-4b87-95a0-218df6659a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:43.304703587Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2740"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:43.486012719Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2767"} {"msg":"wait for service Some(\"oxz_internal_dns_4201b309-328f-4a43-9074-978b08f72f31\") failed: Property not found. retry in 826.962522ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:43.515120671Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_4201b309-328f-4a43-9074-978b08f72f31","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:43.707268607Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2634"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:43.892084332Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2434"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:43.903906672Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3839","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:43.903961503Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3855"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:43.903971123Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:561"} {"msg":"wait for service Some(\"oxz_internal_dns_cec09772-8550-421d-a72a-2ed147e3a7eb\") failed: Property not found. retry in 604.921749ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:43.941806307Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_cec09772-8550-421d-a72a-2ed147e3a7eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:44.463610887Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3863"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:44.47035834Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3919"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:44.572923916Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_internal_dns_ed87b889-28ea-4b87-95a0-218df6659a2b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:44.573032638Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_internal_dns_4201b309-328f-4a43-9074-978b08f72f31"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:44.573039928Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_internal_dns_cec09772-8550-421d-a72a-2ed147e3a7eb"} {"msg":"Writing ledger to /pool/int/9376dea5-4006-4708-b1ab-6628a9deda68/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:44.573084249Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d973a939-4d3f-45e3-a19e-2f06e6f1fe72/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:44.573509435Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:44.573877081Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"3e993f0f-47bf-49f4-aaf1-b925b04f266e","remote_addr":"[fd00:1122:3344:101::1]:46362","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8193359,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:44.576004343Z","hostname":"buskin","pid":1555,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:44.619062697Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3981"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:44.718595687Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3962","rack_subnet":"fd00:1122:3344::"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:44.740358818Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4009"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:44.8379988Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4036"} {"msg":"failed to write DNS configuration (will retry in 130.808263ms)","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:44.931374667Z","hostname":"buskin","pid":1555,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:569","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:44.957756667Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:369"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:44.990352282Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:417"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.02251267Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:656"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.022598061Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:694"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.02315271Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:697","uptime":"3"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.02318775Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:439","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.081785829Z","hostname":"buskin","pid":1555,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.102745327Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.102821879Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.102831089Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.102839239Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.102851609Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.102859319Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.102866489Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.102873609Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.10288033Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.10288963Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.10290943Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.10291873Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.10292451Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.10293327Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.102940301Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.102947861Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.102954821Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.102959531Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.102964661Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.102969801Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.102974891Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.102979781Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.102984631Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.136752954Z","hostname":"buskin","pid":1555,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.183356121Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:585"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.183366371Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.215480199Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.217419538Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.217425668Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: ed87b889-28ea-4b87-95a0-218df6659a2b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a7295e5f-277c-4ceb-8d2f-53dc611d3d65 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7295e5f-277c-4ceb-8d2f-53dc611d3d65 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cec09772-8550-421d-a72a-2ed147e3a7eb, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5fc83c59-f3a1-4ff2-a7e1-d5460eabb2b9 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5fc83c59-f3a1-4ff2-a7e1-d5460eabb2b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 4201b309-328f-4a43-9074-978b08f72f31, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: c8c7ad3c-bcaf-447c-ab38-5a542c9f0928 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8c7ad3c-bcaf-447c-ab38-5a542c9f0928 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01ccde76-5d13-4255-bed6-dba9870e942d, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 54795d16-5339-4df9-9c09-fc1cf6ac673f (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 1a7f9d72-543a-4cb4-aa94-9a8d7355cfcb, kind: Service { id: 01ccde76-5d13-4255-bed6-dba9870e942d }, name: Name(\"ntp-01ccde76-5d13-4255-bed6-dba9870e942d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 198, 193])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.246406768Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.2465185Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:56888"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed87b889-28ea-4b87-95a0-218df6659a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: a7295e5f-277c-4ceb-8d2f-53dc611d3d65 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.246712983Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.246719163Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ed87b889-28ea-4b87-95a0-218df6659a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: a7295e5f-277c-4ceb-8d2f-53dc611d3d65 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cec09772-8550-421d-a72a-2ed147e3a7eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5fc83c59-f3a1-4ff2-a7e1-d5460eabb2b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.268860329Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.2689014Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: cec09772-8550-421d-a72a-2ed147e3a7eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5fc83c59-f3a1-4ff2-a7e1-d5460eabb2b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4201b309-328f-4a43-9074-978b08f72f31, dataset_name: DatasetName { pool_name: ZpoolName { id: c8c7ad3c-bcaf-447c-ab38-5a542c9f0928 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.286832762Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.286874502Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4201b309-328f-4a43-9074-978b08f72f31, dataset_name: DatasetName { pool_name: ZpoolName { id: c8c7ad3c-bcaf-447c-ab38-5a542c9f0928 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.305406454Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/9376dea5-4006-4708-b1ab-6628a9deda68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.305475295Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d973a939-4d3f-45e3-a19e-2f06e6f1fe72/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.305612167Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.305731688Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.316179297Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.348989515Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.351373661Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.351379411Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.381421107Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 198, 193]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.381431808Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1210","zone_type":"ntp"} {"msg":"no nat entry found for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.381772453Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.381777643Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.382058057Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.382460923Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Configuring new Omicron zone: oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.406217694Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:45.424944588Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:46.043284383Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:46.043395095Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d\") failed: Property not found. retry in 39.2461ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:46.171380407Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d\") failed: Property not found. retry in 122.239926ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:46.217886143Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d\") failed: Property not found. retry in 299.24108ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:46.381924013Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d\") failed: Property not found. retry in 216.408785ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:46.693487952Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d\") failed: Property not found. retry in 629.063122ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:46.918449366Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d\") failed: Property not found. retry in 826.813099ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:47.557777979Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:48.381887287Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61704"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:48.382014309Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"4646a030-b95b-49aa-83eb-1b3d6c9cace8","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61704","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":52,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d\") failed: Property not found. retry in 522.686974ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:48.393644836Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d\") failed: Property not found. retry in 659.527555ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:48.925520709Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d\") failed: Property not found. retry in 1.227411987s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:49.594248349Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d\") failed: Property not found. retry in 1.384008773s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:50.830445381Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:52.235217022Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d"} {"msg":"Writing ledger to /pool/int/9376dea5-4006-4708-b1ab-6628a9deda68/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:52.235296274Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d973a939-4d3f-45e3-a19e-2f06e6f1fe72/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:52.235674909Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:52.235941263Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"9db0db69-9a13-4774-b4d2-95b86b6d680f","remote_addr":"[fd00:1122:3344:101::1]:56888","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":6989346,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:52.238236618Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:626"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:52.267549403Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:52.267711395Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:39444"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:52.275694167Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3569"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:52.275867859Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"d7155e14-c0da-4172-884e-42a1ce3fdd16","remote_addr":"[fd00:1122:3344:101::1]:39444","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8022,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:52.278666852Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:52.385874259Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:52.386001861Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:34357"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:56.39505459Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3569"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:56.395188172Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"c919d663-6045-40e8-92aa-70a7ebab4d05","remote_addr":"[fd00:1122:3344:101::1]:34357","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009093,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:56.398289489Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:56.558145776Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:56.558259797Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:55439"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:56.566934769Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3569"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:56.567091451Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"0c0f35cc-a3de-470e-b6fd-ed8fa42d1a36","remote_addr":"[fd00:1122:3344:101::1]:55439","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8728,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:38:56.569189143Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:56.751699673Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:38:56.751805545Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:41464"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_01ccde76-5d13-4255-bed6-dba9870e942d': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:00.759575584Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3569"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:00.759722317Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"c7fa8d97-aba9-40af-87d6-172fc5f71c9d","remote_addr":"[fd00:1122:3344:101::1]:41464","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4007845,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:00.762268465Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:00.885094949Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:00.885221271Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:51858"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:00.893858482Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"dfd4fa32-9f5c-45e7-8881-db37a2db7650","remote_addr":"[fd00:1122:3344:101::1]:51858","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8565,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:00.895768901Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:00.895780542Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:01.047101458Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:01.047190239Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:61231"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:01.054674143Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"1e24c4b6-b80c-4370-98a4-0ff0d70fcb60","remote_addr":"[fd00:1122:3344:101::1]:61231","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":7419,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:01.056585542Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:01.056591552Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:01.257072515Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:01.257177236Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:60212"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:01.26465185Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"e12acce8-a7f6-4978-baf0-aa79f649ec0c","remote_addr":"[fd00:1122:3344:101::1]:60212","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":7409,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:01.266697891Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:01.266713511Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:01.678060735Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:01.678151466Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:63895"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:01.686051026Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"8a6b355b-95a7-43a6-957e-71c620a65e49","remote_addr":"[fd00:1122:3344:101::1]:63895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":7833,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:01.687758002Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:01.687765752Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:02.130971329Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:02.13106209Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:54547"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:02.138155738Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"7c21c99e-d3de-4339-9468-da4229d5e617","remote_addr":"[fd00:1122:3344:101::1]:54547","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":6995,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:02.139536289Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:02.139542359Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:02.803851712Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:02.803938253Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:44142"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:02.811469778Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"7dc4fd79-80ef-433e-b855-5226c9a437c9","remote_addr":"[fd00:1122:3344:101::1]:44142","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":7457,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:02.813809553Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:02.813818113Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:04.27148786Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:04.271584532Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:38614"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:04.279295539Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"dc29c7b3-8ca1-4a94-8200-210c7b5ad1f6","remote_addr":"[fd00:1122:3344:101::1]:38614","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":7638,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:04.281661155Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:04.281668855Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:04.729090206Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:04.729204848Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:43155"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:04.737179419Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"be9d30cd-ef78-4cd9-9b0c-9998f84e23b3","remote_addr":"[fd00:1122:3344:101::1]:43155","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":7903,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:04.739289041Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:04.739300431Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:04.899454702Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"281.395357ms"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:05.365504098Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:05.365591639Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:55086"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:05.373539889Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"0eca4590-49b2-4b42-a1c4-e21039f532e8","remote_addr":"[fd00:1122:3344:101::1]:55086","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":7885,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:05.375784823Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:05.375794624Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:06.191783931Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:06.191875572Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:55205"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:06.199247114Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"325c384a-5649-4968-b97f-e5c24fc406c7","remote_addr":"[fd00:1122:3344:101::1]:55205","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":7305,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:06.201035151Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:06.201058161Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:07.438302599Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:07.438426821Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:42413"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:07.446959481Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"03c0637d-4079-43fc-b95f-1372e43fce0a","remote_addr":"[fd00:1122:3344:101::1]:42413","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8452,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:07.449199345Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:07.449208375Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:09.031813675Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:09.031917507Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:37720"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:09.040795142Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"febb896f-5513-48ee-8df6-c1bc178f6632","remote_addr":"[fd00:1122:3344:101::1]:37720","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8781,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:09.042780822Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:09.042788262Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:10.518964407Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:10.519095859Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:55118"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:10.527563217Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"da0dc51f-e4f0-4d80-8de5-cbe30da27a8f","remote_addr":"[fd00:1122:3344:101::1]:55118","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8382,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:10.52972685Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:10.52973447Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:12.334905129Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:12.335030081Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:46386"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:12.343029662Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"d67709d4-7658-468b-9e86-c60eede3ffa0","remote_addr":"[fd00:1122:3344:101::1]:46386","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":7920,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:12.345550281Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:12.345562131Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:15.178120851Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:15.178252683Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:51883"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:15.187189469Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"1bfdca1a-b6ea-4d97-a3d1-83c4cff1cab5","remote_addr":"[fd00:1122:3344:101::1]:51883","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8864,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: -0.434883267 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:15.190014592Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:15.190024712Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:18.615419703Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:18.615545435Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:45120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:18.624504281Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"9b4a507d-81c7-4006-97a4-283389d1b8c2","remote_addr":"[fd00:1122:3344:101::1]:45120","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8895,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: -0.425548673 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:18.627198282Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:18.627209492Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:19.688721927Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59939"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:19.688820978Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"8c971d0a-3a2d-452d-b552-8bc31881f320","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59939","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":46,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:20.343970709Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"649.325507ms"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:21.776329288Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:21.77644852Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:48574"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:21.784894674Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"455a1af5-032f-4459-ba75-83f048fb4568","remote_addr":"[fd00:1122:3344:101::1]:48574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8375,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: -0.416965753 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:21.787480848Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:21.787492938Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:25.359288067Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:25.359405429Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:57166"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:25.369034105Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"77b7ef1a-2bf1-4787-8aff-9e69be9b3ae2","remote_addr":"[fd00:1122:3344:101::1]:57166","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":9551,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: -0.407232642 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:25.371391351Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:25.371400661Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:31.256977654Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:31.257104116Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:53106"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:31.265801409Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"0f4af1f4-a94d-4b0a-a39f-b6f4121de8cf","remote_addr":"[fd00:1122:3344:101::1]:53106","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8622,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: -0.391218543 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:31.268035543Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:31.268043433Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:35.984895829Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"950.756939ms"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:37.323925887Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:37.324032499Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:54151"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:37.332884154Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"137fc8f5-84a2-42cc-86de-36d035a8de98","remote_addr":"[fd00:1122:3344:101::1]:54151","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8782,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: -0.374742568 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:37.335509254Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:37.335520344Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:46.421427181Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:46.421550754Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:43617"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:46.4305032Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"db79f488-9a40-46d9-9427-2ee1a30e306e","remote_addr":"[fd00:1122:3344:101::1]:43617","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8868,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: -0.350036234 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:46.432755814Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:46.432765134Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:49.639404736Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50598"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:49.639490067Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"2e8c036b-951a-45a2-8515-3a99c87ee08c","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50598","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":45,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:51.926387971Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"1.773632663s"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:57.742636168Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:57.74277862Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:48299"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:57.752288885Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"02c3982a-eaeb-4993-90d3-904f4137ff4c","remote_addr":"[fd00:1122:3344:101::1]:48299","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":9497,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: -0.319289923 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:39:57.754788983Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:39:57.754797634Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:40:05.930418198Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:40:05.930581951Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:51343"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:40:05.939265593Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"16370c49-1f68-45d6-a3b1-ba3171b311e7","remote_addr":"[fd00:1122:3344:101::1]:51343","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8663,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: -0.297056407 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:40:05.941931833Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:40:05.941943414Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:40:08.688379402Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"2.097217284s"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:40:13.800551394Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:40:13.800660715Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:50005"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:40:13.809822555Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"adc62ad3-8ea5-4d83-a5d1-8ab4d9bb9902","remote_addr":"[fd00:1122:3344:101::1]:50005","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":9090,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: -0.277175993 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:40:13.812208141Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:40:13.812221041Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:40:19.591605148Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35344"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:40:19.592607933Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"909c93b1-a739-4d9c-8eed-c7070cf15900","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35344","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":959,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:40:25.773667402Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"7.838419463s"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:40:30.964026462Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:40:30.964155817Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:41865"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:40:30.973240465Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"4b3e8cca-ab77-4bc4-bec6-c2e267707aca","remote_addr":"[fd00:1122:3344:101::1]:41865","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":9026,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: -0.230565161 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:40:30.975744733Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:40:30.975754873Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:40:46.797823799Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:40:46.797946431Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:61369"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:40:46.806497501Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"6827cf56-ed86-4373-845b-045e45f56917","remote_addr":"[fd00:1122:3344:101::1]:61369","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8507,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: -0.187566668 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:40:46.808832537Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:40:46.808863367Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:40:48.586143877Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"20.314330503s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:40:49.543321953Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65377"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:40:49.543404385Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"d33d4b09-8097-492b-8d9c-6d7e62f51881","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65377","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":39,"response_code":"200"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:40:59.360752564Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:40:59.360874316Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:55505"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:40:59.369723951Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"d3e61596-1c2a-4596-9e43-a9a24f334411","remote_addr":"[fd00:1122:3344:101::1]:55505","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8801,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: -0.153449073 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:40:59.372102567Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:40:59.372112727Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:41:19.493079276Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37382"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:41:19.493167028Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"0942b2ba-15af-4a6e-afdc-357b0842a0cf","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37382","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":43,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:41:23.840148124Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"28.871059863s"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:41:29.378089515Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:41:29.378218887Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:33581"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:41:29.38629585Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"2f209a2e-f870-49f8-b4ba-7d522fb2c1ba","remote_addr":"[fd00:1122:3344:101::1]:33581","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8029,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: -0.075160898 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:41:29.389414538Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:41:29.389427808Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:41:45.216579369Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:41:45.216750511Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:40478"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:41:45.225636627Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"2e076ce1-92a9-4ed7-86c4-9fc5111deba5","remote_addr":"[fd00:1122:3344:101::1]:40478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8865,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: -0.032146476 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:41:45.228684933Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:41:45.228694503Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:41:49.444142768Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34255"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:41:49.44424587Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"6a4b73b3-f9ab-4458-a2bb-1437599dc1d8","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34255","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":56,"response_code":"200"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:42:03.545030272Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:42:03.545148974Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:39991"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:42:03.553625942Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"e71948ea-6a47-43bb-bf3c-62e31f865eb7","remote_addr":"[fd00:1122:3344:101::1]:39991","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8393,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 2.538e-6 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:42:03.555932477Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:42:03.555940618Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:42:07.653291025Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"57.046111302s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:42:19.45239901Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51266"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:42:19.452495991Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"a52fe4dd-e412-4bf7-a4cb-60a8631fd637","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51266","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47,"response_code":"200"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:42:30.058716497Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:42:30.058830409Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:55878"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:42:30.067974687Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"0144e8db-acd4-4da4-8ce4-229b39625076","remote_addr":"[fd00:1122:3344:101::1]:55878","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8964,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.004849952 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:42:30.069665903Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:42:30.069672993Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:42:49.485431629Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58386"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:42:49.48554782Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"526882e1-5f2c-43b5-92cf-140a0079cfb0","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58386","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":70,"response_code":"200"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:04.53608842Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:04.536215582Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:63107"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:04.545091757Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"ed102788-c88b-40d7-8480-287f2a43e279","remote_addr":"[fd00:1122:3344:101::1]:63107","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8797,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0044759 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:04.547470733Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:04.547518183Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:19.520839482Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37075"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:19.520894703Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"4f139a49-dd68-4825-94fd-d1cc5021fbfe","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37075","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":33,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:19.733812634Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"d7d0bedb-61a2-44cf-b5a5-637dc79ea041","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"85.642898578s"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:37.193379445Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.115140822Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.115265313Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:39088"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.136272292Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"c07593f9-82ca-4bca-b8bb-bcb9cb41ea81","remote_addr":"[fd00:1122:3344:101::1]:39088","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":20848,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 1209934681, ip_addr: 72.30.35.89, stratum: 3, ref_time: 1721335402.4834805, correction: 0.006478171 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.139199337Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.139206447Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1224"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: ed87b889-28ea-4b87-95a0-218df6659a2b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: a7295e5f-277c-4ceb-8d2f-53dc611d3d65 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7295e5f-277c-4ceb-8d2f-53dc611d3d65 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: cec09772-8550-421d-a72a-2ed147e3a7eb, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 5fc83c59-f3a1-4ff2-a7e1-d5460eabb2b9 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5fc83c59-f3a1-4ff2-a7e1-d5460eabb2b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 4201b309-328f-4a43-9074-978b08f72f31, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: c8c7ad3c-bcaf-447c-ab38-5a542c9f0928 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8c7ad3c-bcaf-447c-ab38-5a542c9f0928 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01ccde76-5d13-4255-bed6-dba9870e942d, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 54795d16-5339-4df9-9c09-fc1cf6ac673f (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 1a7f9d72-543a-4cb4-aa94-9a8d7355cfcb, kind: Service { id: 01ccde76-5d13-4255-bed6-dba9870e942d }, name: Name(\"ntp-01ccde76-5d13-4255-bed6-dba9870e942d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 198, 193])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 730d64ca-9ebe-405d-9ed2-5f48d9248ea1, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: a7295e5f-277c-4ceb-8d2f-53dc611d3d65 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7295e5f-277c-4ceb-8d2f-53dc611d3d65 (zpool), kind: External } } } }, OmicronZoneConfig { id: b34a3259-5a67-4828-90b6-85d3bd3357b8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 5fc83c59-f3a1-4ff2-a7e1-d5460eabb2b9 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5fc83c59-f3a1-4ff2-a7e1-d5460eabb2b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f119905-fd2b-4981-9885-1ae43f417871, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: c8c7ad3c-bcaf-447c-ab38-5a542c9f0928 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8c7ad3c-bcaf-447c-ab38-5a542c9f0928 (zpool), kind: External } } } }, OmicronZoneConfig { id: 476c0826-fa51-40d2-ba6d-84998a297494, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 61b850b7-5d2e-4dcf-b938-603fca674806 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 61b850b7-5d2e-4dcf-b938-603fca674806 (zpool), kind: External } } } }, OmicronZoneConfig { id: b32790a6-3f39-4812-9473-ee7832e15b77, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 041bb4d1-3950-4b3e-8f1e-68d1fe20601c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 041bb4d1-3950-4b3e-8f1e-68d1fe20601c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.168154256Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.168318769Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:63715"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed87b889-28ea-4b87-95a0-218df6659a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: a7295e5f-277c-4ceb-8d2f-53dc611d3d65 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.168772936Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.168787356Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ed87b889-28ea-4b87-95a0-218df6659a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: a7295e5f-277c-4ceb-8d2f-53dc611d3d65 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cec09772-8550-421d-a72a-2ed147e3a7eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5fc83c59-f3a1-4ff2-a7e1-d5460eabb2b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.189791655Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.189840955Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: cec09772-8550-421d-a72a-2ed147e3a7eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5fc83c59-f3a1-4ff2-a7e1-d5460eabb2b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4201b309-328f-4a43-9074-978b08f72f31, dataset_name: DatasetName { pool_name: ZpoolName { id: c8c7ad3c-bcaf-447c-ab38-5a542c9f0928 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.208248115Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.208292985Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4201b309-328f-4a43-9074-978b08f72f31, dataset_name: DatasetName { pool_name: ZpoolName { id: c8c7ad3c-bcaf-447c-ab38-5a542c9f0928 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 730d64ca-9ebe-405d-9ed2-5f48d9248ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: a7295e5f-277c-4ceb-8d2f-53dc611d3d65 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.224554232Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.224596033Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 730d64ca-9ebe-405d-9ed2-5f48d9248ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: a7295e5f-277c-4ceb-8d2f-53dc611d3d65 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b34a3259-5a67-4828-90b6-85d3bd3357b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 5fc83c59-f3a1-4ff2-a7e1-d5460eabb2b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.276382149Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.27643642Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b34a3259-5a67-4828-90b6-85d3bd3357b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 5fc83c59-f3a1-4ff2-a7e1-d5460eabb2b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f119905-fd2b-4981-9885-1ae43f417871, dataset_name: DatasetName { pool_name: ZpoolName { id: c8c7ad3c-bcaf-447c-ab38-5a542c9f0928 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.327609976Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.327659447Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3f119905-fd2b-4981-9885-1ae43f417871, dataset_name: DatasetName { pool_name: ZpoolName { id: c8c7ad3c-bcaf-447c-ab38-5a542c9f0928 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 476c0826-fa51-40d2-ba6d-84998a297494, dataset_name: DatasetName { pool_name: ZpoolName { id: 61b850b7-5d2e-4dcf-b938-603fca674806 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.376002351Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.376048542Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 476c0826-fa51-40d2-ba6d-84998a297494, dataset_name: DatasetName { pool_name: ZpoolName { id: 61b850b7-5d2e-4dcf-b938-603fca674806 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32790a6-3f39-4812-9473-ee7832e15b77, dataset_name: DatasetName { pool_name: ZpoolName { id: 041bb4d1-3950-4b3e-8f1e-68d1fe20601c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.429049356Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.429098827Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b32790a6-3f39-4812-9473-ee7832e15b77, dataset_name: DatasetName { pool_name: ZpoolName { id: 041bb4d1-3950-4b3e-8f1e-68d1fe20601c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.479068085Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/9376dea5-4006-4708-b1ab-6628a9deda68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.479134686Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d973a939-4d3f-45e3-a19e-2f06e6f1fe72/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.479340749Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.479442551Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.564097636Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.564178327Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.564185537Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.564191847Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.564197927Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.6533351Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.65727627Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.66125988Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.665961092Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.671066419Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.676068585Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.680256329Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.684521693Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.688738464Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:47.693125284Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:49.554327411Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42770"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:49.555266966Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"25b32620-7d62-47b2-9b6c-c0cf30d877ef","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42770","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":853,"response_code":"200"} {"msg":"Profile for oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:50.563355826Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:50.563743162Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:50.569762513Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:50.570122119Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:50.570135779Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:50.575165105Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1\") failed: Property not found. retry in 65.184119ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:50.710580631Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494\") failed: Property not found. retry in 45.999591ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:50.717602517Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8\") failed: Property not found. retry in 74.290411ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:50.723801431Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:50.726261898Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:50.726560763Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:50.732788317Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:43:50.732906819Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494\") failed: Property not found. retry in 147.388851ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:50.779136861Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1\") failed: Property not found. retry in 130.498149ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:50.888233187Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8\") failed: Property not found. retry in 74.17623ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:51.135072313Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494\") failed: Property not found. retry in 211.811151ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:51.359458809Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8\") failed: Property not found. retry in 206.219582ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:51.561399166Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1\") failed: Property not found. retry in 194.988993ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:51.636515164Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494\") failed: Property not found. retry in 241.617121ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:51.703939537Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8\") failed: Property not found. retry in 347.587139ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:52.218716249Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494\") failed: Property not found. retry in 1.124387698s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:52.274298043Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77\") failed: Property not found. retry in 42.793521ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:52.321988517Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871\") failed: Property not found. retry in 70.939659ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:52.410637822Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1\") failed: Property not found. retry in 511.668014ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:52.448374795Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77\") failed: Property not found. retry in 57.502337ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:52.467210141Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871\") failed: Property not found. retry in 58.730733ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:52.522202696Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77\") failed: Property not found. retry in 232.772013ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:52.544700217Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8\") failed: Property not found. retry in 874.398172ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:52.583654808Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871\") failed: Property not found. retry in 101.560883ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:52.598245Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871\") failed: Property not found. retry in 564.059332ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:52.718934842Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77\") failed: Property not found. retry in 299.925447ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:52.799503975Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1\") failed: Property not found. retry in 815.795763ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:52.974109884Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77\") failed: Property not found. retry in 571.797439ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:53.123853997Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871\") failed: Property not found. retry in 1.095261229s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:53.298560419Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494\") failed: Property not found. retry in 1.002844674s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:53.417981951Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8\") failed: Property not found. retry in 778.318301ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:53.476492979Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77\") failed: Property not found. retry in 963.664447ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:53.711654208Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1\") failed: Property not found. retry in 765.57293ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:53.802346125Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8\") failed: Property not found. retry in 1.432253831s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:54.264052232Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871\") failed: Property not found. retry in 791.300602ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:54.401859993Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494\") failed: Property not found. retry in 616.555127ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:54.429386399Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1\") failed: Property not found. retry in 1.47358918s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:54.5762586Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77\") failed: Property not found. retry in 1.099424947s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:54.684497913Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494\") failed: Property not found. retry in 1.351642445s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:55.055082888Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871\") failed: Property not found. retry in 1.278861262s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:55.202970952Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8\") failed: Property not found. retry in 532.337512ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:55.706136679Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77\") failed: Property not found. retry in 1.326498519s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:55.793258632Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1\") failed: Property not found. retry in 960.148218ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:56.059194628Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8\") failed: Property not found. retry in 584.922266ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:56.248824396Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494\") failed: Property not found. retry in 951.947494ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:56.416327248Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871\") failed: Property not found. retry in 755.560429ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:56.490405992Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8\") failed: Property not found. retry in 1.331341181s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:56.844820021Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1\") failed: Property not found. retry in 1.436955558s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:57.029770859Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77\") failed: Property not found. retry in 798.581643ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:57.129002834Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871\") failed: Property not found. retry in 970.303525ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:57.255597996Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494\") failed: Property not found. retry in 988.153837ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:57.37842312Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77\") failed: Property not found. retry in 586.966234ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:57.938542241Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8\") failed: Property not found. retry in 1.128300557s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:58.186126409Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871\") failed: Property not found. retry in 1.372565352s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:58.235402427Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494\") failed: Property not found. retry in 1.386647697s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:58.376519189Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1\") failed: Property not found. retry in 1.162408387s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:58.475886077Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77\") failed: Property not found. retry in 1.152969524s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:58.535688065Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8\") failed: Property not found. retry in 813.15816ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:59.325184297Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871\") failed: Property not found. retry in 873.24108ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:59.617336632Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1\") failed: Property not found. retry in 986.44083ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:59.648101399Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77\") failed: Property not found. retry in 596.085753ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:59.698003106Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494\") failed: Property not found. retry in 546.835583ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:43:59.772009619Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8\") failed: Property not found. retry in 967.24942ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:44:00.149209374Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77\") failed: Property not found. retry in 1.224597619s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:44:00.303578207Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494\") failed: Property not found. retry in 1.321040817s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:44:00.327817885Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871\") failed: Property not found. retry in 1.005320713s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:44:00.500423024Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1\") failed: Property not found. retry in 859.760979ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:44:00.64442286Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8\") failed: Property not found. retry in 1.094807649s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:44:01.125808095Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1\") failed: Property not found. retry in 715.259325ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:44:01.513900926Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871\") failed: Property not found. retry in 1.068643367s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:44:01.523456571Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494\") failed: Property not found. retry in 1.43643977s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:44:01.657596077Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8\") failed: Property not found. retry in 1.440050206s","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:44:02.232899958Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1\") failed: Property not found. retry in 513.858142ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:44:02.242153479Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:44:03.694087336Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_cockroachdb_476c0826-fa51-40d2-ba6d-84998a297494"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:44:03.694180847Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_cockroachdb_b32790a6-3f39-4812-9473-ee7832e15b77"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:44:03.694191097Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:44:03.694201597Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_cockroachdb_b34a3259-5a67-4828-90b6-85d3bd3357b8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:44:03.694206497Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_cockroachdb_730d64ca-9ebe-405d-9ed2-5f48d9248ea1"} {"msg":"Writing ledger to /pool/int/9376dea5-4006-4708-b1ab-6628a9deda68/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:44:03.694219938Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d973a939-4d3f-45e3-a19e-2f06e6f1fe72/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:44:03.694618674Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:44:03.694895198Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"8dab4aa4-a934-4b23-bdc1-bd1a6571644a","remote_addr":"[fd00:1122:3344:101::1]:63715","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":16525909,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:44:03.73325533Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:49202"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:44:03.762064207Z","hostname":"buskin","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"9c72acd9-fb79-43f6-a9ca-7d41c4a79c85","remote_addr":"[fd00:1122:3344:101::1]:49202","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":28682,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:44:03.762348372Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:966","retry_after":"121.877792ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9c72acd9-fb79-43f6-a9ca-7d41c4a79c85\", \"content-length\": \"124\", \"date\": \"Thu, 18 Jul 2024 20:44:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9c72acd9-fb79-43f6-a9ca-7d41c4a79c85\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:44:03.912118445Z","hostname":"buskin","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"e03f304a-e176-40fb-897c-9bd769caf22b","remote_addr":"[fd00:1122:3344:101::1]:49202","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":27562,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:44:03.912296667Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:966","retry_after":"67.320173ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e03f304a-e176-40fb-897c-9bd769caf22b\", \"content-length\": \"124\", \"date\": \"Thu, 18 Jul 2024 20:44:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e03f304a-e176-40fb-897c-9bd769caf22b\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:44:04.011224219Z","hostname":"buskin","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"79f12506-e435-4092-bc7b-7926ddfba71d","remote_addr":"[fd00:1122:3344:101::1]:49202","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":29689,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:44:04.011420272Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:966","retry_after":"97.879578ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"79f12506-e435-4092-bc7b-7926ddfba71d\", \"content-length\": \"124\", \"date\": \"Thu, 18 Jul 2024 20:44:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"79f12506-e435-4092-bc7b-7926ddfba71d\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:44:04.13581054Z","hostname":"buskin","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"91e88c90-2f51-42ec-be14-5ef13e61f3ee","remote_addr":"[fd00:1122:3344:101::1]:49202","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":26164,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:44:04.135989463Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:966","retry_after":"153.68896ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"91e88c90-2f51-42ec-be14-5ef13e61f3ee\", \"content-length\": \"124\", \"date\": \"Thu, 18 Jul 2024 20:44:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"91e88c90-2f51-42ec-be14-5ef13e61f3ee\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:44:04.316018455Z","hostname":"buskin","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"2eb5cd9e-544e-43ee-911f-b1413d08e9f6","remote_addr":"[fd00:1122:3344:101::1]:49202","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":24807,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:44:04.316187768Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:966","retry_after":"277.105975ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2eb5cd9e-544e-43ee-911f-b1413d08e9f6\", \"content-length\": \"124\", \"date\": \"Thu, 18 Jul 2024 20:44:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2eb5cd9e-544e-43ee-911f-b1413d08e9f6\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:44:04.620311353Z","hostname":"buskin","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"48ce4f3b-3dbb-4bbe-8b20-92989c3b36c1","remote_addr":"[fd00:1122:3344:101::1]:49202","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":25811,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:44:04.620482946Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:966","retry_after":"209.593701ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"48ce4f3b-3dbb-4bbe-8b20-92989c3b36c1\", \"content-length\": \"124\", \"date\": \"Thu, 18 Jul 2024 20:44:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"48ce4f3b-3dbb-4bbe-8b20-92989c3b36c1\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:44:04.857679745Z","hostname":"buskin","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"7d9bbae7-63d2-4afe-bcec-49f32d7c4655","remote_addr":"[fd00:1122:3344:101::1]:49202","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":26861,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:44:04.857862828Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:966","retry_after":"332.326635ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7d9bbae7-63d2-4afe-bcec-49f32d7c4655\", \"content-length\": \"124\", \"date\": \"Thu, 18 Jul 2024 20:44:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7d9bbae7-63d2-4afe-bcec-49f32d7c4655\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:44:05.219160842Z","hostname":"buskin","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"461a9ee4-9773-446c-9aa9-a45d737fccbe","remote_addr":"[fd00:1122:3344:101::1]:49202","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":27930,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:44:05.219367645Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:966","retry_after":"392.78367ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"461a9ee4-9773-446c-9aa9-a45d737fccbe\", \"content-length\": \"124\", \"date\": \"Thu, 18 Jul 2024 20:44:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"461a9ee4-9773-446c-9aa9-a45d737fccbe\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:44:05.640976104Z","hostname":"buskin","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"45b1cef0-1156-4420-b1d6-bc696932fa62","remote_addr":"[fd00:1122:3344:101::1]:49202","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_3f119905-fd2b-4981-9885-1ae43f417871 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":28215,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-07-18T20:44:05.641203777Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:966","retry_after":"490.336896ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"45b1cef0-1156-4420-b1d6-bc696932fa62\", \"content-length\": \"124\", \"date\": \"Thu, 18 Jul 2024 20:44:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"45b1cef0-1156-4420-b1d6-bc696932fa62\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::5]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:44:06.167927481Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3403"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-07-18T20:44:20.864426126Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3421"}