[ May 20 16:05:21 Enabled. ] [ May 20 16:05:21 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ May 20 16:05:21 Method "start" exited with status 0. ] [ May 20 16:05:21 Rereading configuration. ] [ May 20 16:05:21 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:21.875828009Z","hostname":"buskin","pid":1562,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:21.875999334Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:21.928302869Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:21.92834618Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:22.041128274Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:22.04995973Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:22.057893278Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:22.666885689Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:9fb2::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:22.803752956Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:119"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:22.803795197Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:131"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:22.803924101Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:22.803980173Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:162","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:22.803991123Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:506"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:22.804013334Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:625"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:22.806841512Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:22.806848733Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:674"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:22.806873373Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:22.806880183Z","hostname":"buskin","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:733"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:22.806915095Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:22.806959596Z","hostname":"buskin","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:22.807053719Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:22.807070729Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:22.807112361Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:22.818774605Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:22.818798996Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"266edb61-c6f9-4597-9662-11a7fcae56f7 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:22.916664644Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 266edb61-c6f9-4597-9662-11a7fcae56f7 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.130032591Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.130076432Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.130083402Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.130095903Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.130102673Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.130107333Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.130125914Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.138785094Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.138805835Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"c347173a-b2f9-4fe0-9e29-72f0b24d1c5f (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.227062503Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: c347173a-b2f9-4fe0-9e29-72f0b24d1c5f (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433398001Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433450662Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 266edb61-c6f9-4597-9662-11a7fcae56f7 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c347173a-b2f9-4fe0-9e29-72f0b24d1c5f (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433472993Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433483823Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433488584Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433492754Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433500394Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433536015Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 266edb61-c6f9-4597-9662-11a7fcae56f7 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c347173a-b2f9-4fe0-9e29-72f0b24d1c5f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433549885Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433556026Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433559886Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433562966Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433597297Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_2.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433611358Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433614858Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 266edb61-c6f9-4597-9662-11a7fcae56f7 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c347173a-b2f9-4fe0-9e29-72f0b24d1c5f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433634548Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433638978Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433643388Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433646568Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_3.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433650309Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433656669Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433660319Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 266edb61-c6f9-4597-9662-11a7fcae56f7 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c347173a-b2f9-4fe0-9e29-72f0b24d1c5f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433671289Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433675229Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.4336791Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_4.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.43368464Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.43368807Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.43369445Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 266edb61-c6f9-4597-9662-11a7fcae56f7 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c347173a-b2f9-4fe0-9e29-72f0b24d1c5f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433715311Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433719281Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433723291Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_5.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433727391Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433730271Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433743202Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 266edb61-c6f9-4597-9662-11a7fcae56f7 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c347173a-b2f9-4fe0-9e29-72f0b24d1c5f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433756532Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433763822Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433766932Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433781653Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 266edb61-c6f9-4597-9662-11a7fcae56f7 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c347173a-b2f9-4fe0-9e29-72f0b24d1c5f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433805404Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433810344Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433813344Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433825314Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 266edb61-c6f9-4597-9662-11a7fcae56f7 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c347173a-b2f9-4fe0-9e29-72f0b24d1c5f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433845325Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433849505Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433852815Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433859115Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433862325Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433866075Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433869235Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433879336Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433883406Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433887456Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 266edb61-c6f9-4597-9662-11a7fcae56f7 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c347173a-b2f9-4fe0-9e29-72f0b24d1c5f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433905027Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:91"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433914687Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:93"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433918817Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433922027Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 266edb61-c6f9-4597-9662-11a7fcae56f7 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c347173a-b2f9-4fe0-9e29-72f0b24d1c5f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433956098Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433960108Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433963438Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433967029Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.433970449Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"No ledger in /pool/int/266edb61-c6f9-4597-9662-11a7fcae56f7/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.4340042Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c347173a-b2f9-4fe0-9e29-72f0b24d1c5f/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.43402093Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.434034491Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/266edb61-c6f9-4597-9662-11a7fcae56f7/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.434041771Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c347173a-b2f9-4fe0-9e29-72f0b24d1c5f/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.434046861Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.434051571Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.434079722Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.434088012Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.444601671Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:711"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.444629852Z","hostname":"buskin","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999922348s","next_cleanup":"Instant { tv_sec: 1471, tv_nsec: 176492574 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.444648432Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.444660723Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/266edb61-c6f9-4597-9662-11a7fcae56f7/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.444720965Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c347173a-b2f9-4fe0-9e29-72f0b24d1c5f/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.447393188Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.44778257Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:196"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.447855832Z","hostname":"buskin","pid":1562,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV1 {\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV1 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n },\n ],\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.1.174,\n prefix: 24,\n },\n ),\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.448139101Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1003"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.448168772Z","hostname":"buskin","pid":1562,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.448244575Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/266edb61-c6f9-4597-9662-11a7fcae56f7/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.448317207Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c347173a-b2f9-4fe0-9e29-72f0b24d1c5f/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.448349888Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.448354798Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1063"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.448391919Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/266edb61-c6f9-4597-9662-11a7fcae56f7/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.44842231Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c347173a-b2f9-4fe0-9e29-72f0b24d1c5f/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.448442081Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.448450411Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1104"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.448455921Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.448460342Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.448497953Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/266edb61-c6f9-4597-9662-11a7fcae56f7/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.448525513Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c347173a-b2f9-4fe0-9e29-72f0b24d1c5f/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.448907845Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.449067861Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.449091731Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1137"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.449138183Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/266edb61-c6f9-4597-9662-11a7fcae56f7/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.449147273Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c347173a-b2f9-4fe0-9e29-72f0b24d1c5f/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.449297918Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.449443382Z","hostname":"buskin","pid":1562,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.449488194Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 574b7f2f-83a4-499c-82e5-b830eda888e0, rack_id: b76798f0-832d-4d04-bf6b-1df5e9714347, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.449625128Z","hostname":"buskin","pid":1562,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:60258","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 574b7f2f-83a4-499c-82e5-b830eda888e0, rack_id: b76798f0-832d-4d04-bf6b-1df5e9714347, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.449727301Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:360"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.449738351Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:375"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.449763652Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/server.rs:44"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.449773412Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.449777123Z","hostname":"buskin","pid":1562,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.449783483Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:530"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.449787153Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"No ledger in /pool/int/266edb61-c6f9-4597-9662-11a7fcae56f7/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.449814134Z","hostname":"buskin","pid":1562,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c347173a-b2f9-4fe0-9e29-72f0b24d1c5f/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.449848525Z","hostname":"buskin","pid":1562,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.449856155Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:519"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.449859805Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:547"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.452386504Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3849"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.460044424Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.469632443Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.48008251Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.489676979Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.498431893Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.507968331Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.516807217Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.520873064Z","hostname":"buskin","pid":1562,"sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:355"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.520914186Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 86.840344ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.521451332Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"31d316b5-a710-432f-9361-5773fcb2e3cc\", \"content-length\": \"124\", \"date\": \"Mon, 20 May 2024 16:05:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"31d316b5-a710-432f-9361-5773fcb2e3cc\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.525418736Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.534418188Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"No disks to be automatically managed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.534464849Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:558"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.53447963Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.53448505Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.5344904Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:370"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.53449789Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.53450552Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:449","result":"DisksManagementResult { status: [] }"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.534520081Z","hostname":"buskin","pid":1562,"sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:367"} {"msg":"attempting to create encrypted zvol: name=\"oxi_266edb61-c6f9-4597-9662-11a7fcae56f7/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.546641299Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.557899621Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"successfully created encrypted zvol: name=\"oxi_266edb61-c6f9-4597-9662-11a7fcae56f7/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.569418961Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_266edb61-c6f9-4597-9662-11a7fcae56f7/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.569456892Z","hostname":"buskin","pid":1562,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.574042436Z","hostname":"buskin","pid":1562,"sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:382"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.574051126Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_266edb61-c6f9-4597-9662-11a7fcae56f7/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.574065507Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.576310067Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.609893066Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 89.537861ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.610952139Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"76fd85d8-5742-4ed0-9ed0-f15257a3d76a\", \"content-length\": \"124\", \"date\": \"Mon, 20 May 2024 16:05:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"76fd85d8-5742-4ed0-9ed0-f15257a3d76a\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.648062559Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.701407156Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 126.629628ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.701935382Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"aa663869-50e9-4254-9b1e-01180e08a625\", \"content-length\": \"124\", \"date\": \"Mon, 20 May 2024 16:05:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"aa663869-50e9-4254-9b1e-01180e08a625\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.829990834Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 218.933371ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.830462899Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"69cca519-5678-4ec5-b0af-79922c7a6a7c\", \"content-length\": \"124\", \"date\": \"Mon, 20 May 2024 16:05:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"69cca519-5678-4ec5-b0af-79922c7a6a7c\" }"} {"msg":"Mounting oxi_266edb61-c6f9-4597-9662-11a7fcae56f7/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.867151805Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.88041729Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.880503322Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.880535593Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:23.880565284Z","hostname":"buskin","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.050579277Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 132.320462ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.054009824Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"afee50df-073f-40b4-b34b-de09f9bb97bb\", \"content-length\": \"124\", \"date\": \"Mon, 20 May 2024 16:05:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"afee50df-073f-40b4-b34b-de09f9bb97bb\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.188028292Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 267.63505ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.18890852Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"db065b10-d24b-4dae-ac41-a2a0a6ba3f4a\", \"content-length\": \"124\", \"date\": \"Mon, 20 May 2024 16:05:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"db065b10-d24b-4dae-ac41-a2a0a6ba3f4a\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.457980478Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 174.173298ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.458607367Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"34559f85-cefe-4f34-9ff7-63ea8699e550\", \"content-length\": \"124\", \"date\": \"Mon, 20 May 2024 16:05:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"34559f85-cefe-4f34-9ff7-63ea8699e550\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.577040758Z","hostname":"buskin","pid":1562,"sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.577643497Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:56176","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:196","component":"producer-server","component":"MetricsManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:196"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.577674428Z","hostname":"buskin","pid":1562,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.577756711Z","hostname":"buskin","pid":1562,"file":"oximeter/producer/src/lib.rs:276","component":"producer-server","component":"MetricsManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:276","interval":"30s","address":"[fd00:1122:3344:101::1]:56176","producer_id":"574b7f2f-83a4-499c-82e5-b830eda888e0"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:24.583454429Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"272.730214ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.622905251Z","hostname":"buskin","pid":1562,"component":"VmmReservoirManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.622933252Z","hostname":"buskin","pid":1562,"component":"VmmReservoirManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.623003445Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:993","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.623012845Z","hostname":"buskin","pid":1562,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.623688996Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:196"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.623726037Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:430"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.623754688Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/266edb61-c6f9-4597-9662-11a7fcae56f7/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.623773679Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c347173a-b2f9-4fe0-9e29-72f0b24d1c5f/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.624057927Z","hostname":"buskin","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.624231933Z","hostname":"buskin","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.624250964Z","hostname":"buskin","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.1.100, vlan_id: None }], addresses: [V4(Ipv4Network { addr: 10.151.1.174, prefix: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.624259734Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.624280105Z","hostname":"buskin","pid":1562,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:60258","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.823269963Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.823349445Z","hostname":"buskin","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.823704006Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.823781759Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"No ledger in /pool/int/266edb61-c6f9-4597-9662-11a7fcae56f7/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.823795029Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c347173a-b2f9-4fe0-9e29-72f0b24d1c5f/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.82380974Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.823847391Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.823866141Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/574b7f2f-83a4-499c-82e5-b830eda888e0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:24.828401353Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.855660175Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:55541"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.855848631Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.855945454Z","hostname":"buskin","pid":1562,"uri":"/inventory","method":"GET","req_id":"688eba74-e4f7-471c-9112-9ed69f91cc9b","remote_addr":"[fd00:1122:3344:101::1]:55541","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":168,"response_code":"200"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:24.858030919Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"437.252876ms"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.886012733Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:47336"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.886121487Z","hostname":"buskin","pid":1562,"uri":"/sled-role","method":"GET","req_id":"79d44106-cec0-442a-9671-8ac109dbb67c","remote_addr":"[fd00:1122:3344:101::1]:47336","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":43,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.88752268Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/266edb61-c6f9-4597-9662-11a7fcae56f7/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.887556652Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c347173a-b2f9-4fe0-9e29-72f0b24d1c5f/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.887856931Z","hostname":"buskin","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.888170541Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:815"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: b0888b44-6d56-47bc-80bc-121ef1c46e57, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, pool_id: af4109c5-bbac-4d47-a0e2-4b1267a1cb61 (zpool) }, OmicronPhysicalDiskConfig { id: 8d1bb324-88ed-4651-93af-cd2230acee00, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, pool_id: 6299fefa-5c12-4270-b629-f723a7cb7885 (zpool) }, OmicronPhysicalDiskConfig { id: 4bba3622-5bff-45cc-b184-347b48436e83, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, pool_id: 8063aaf6-9dd7-4b2a-b096-66bd421c61e1 (zpool) }, OmicronPhysicalDiskConfig { id: 84f4c0c7-104f-4249-8274-8002380cb3cb, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, pool_id: 0fd8bd93-1766-40ff-af7d-07d69a799d88 (zpool) }, OmicronPhysicalDiskConfig { id: a33474a8-89de-4199-80e4-9ebf16164a33, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, pool_id: c02d0d46-c9bc-47f2-991b-6cec4937c8f3 (zpool) }, OmicronPhysicalDiskConfig { id: 79cf7afe-0936-4401-9b3c-65dd7c9e40a1, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, pool_id: b2297356-8cd5-4606-999a-8889c6613486 (zpool) }, OmicronPhysicalDiskConfig { id: 3d6ac1ad-2b70-489d-8490-6a664c1e4228, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, pool_id: 459e4cea-0570-4a0f-b58a-37b154241edb (zpool) }, OmicronPhysicalDiskConfig { id: d0994444-2f38-4a5f-ac4c-b7db579a55db, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, pool_id: 0c51412c-3c0e-4ec7-9e61-170beed202ad (zpool) }, OmicronPhysicalDiskConfig { id: da3d1d5a-51f9-4d4f-9f77-b836644e1c54, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, pool_id: 29e61161-eea7-4c3e-8e3e-8d8a030f4b7b (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.920890663Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:347"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.920969906Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:62347"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: b0888b44-6d56-47bc-80bc-121ef1c46e57, pool_id: af4109c5-bbac-4d47-a0e2-4b1267a1cb61 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 8d1bb324-88ed-4651-93af-cd2230acee00, pool_id: 6299fefa-5c12-4270-b629-f723a7cb7885 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 4bba3622-5bff-45cc-b184-347b48436e83, pool_id: 8063aaf6-9dd7-4b2a-b096-66bd421c61e1 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 84f4c0c7-104f-4249-8274-8002380cb3cb, pool_id: 0fd8bd93-1766-40ff-af7d-07d69a799d88 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: a33474a8-89de-4199-80e4-9ebf16164a33, pool_id: c02d0d46-c9bc-47f2-991b-6cec4937c8f3 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 79cf7afe-0936-4401-9b3c-65dd7c9e40a1, pool_id: b2297356-8cd5-4606-999a-8889c6613486 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 3d6ac1ad-2b70-489d-8490-6a664c1e4228, pool_id: 459e4cea-0570-4a0f-b58a-37b154241edb (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: d0994444-2f38-4a5f-ac4c-b7db579a55db, pool_id: 0c51412c-3c0e-4ec7-9e61-170beed202ad (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: da3d1d5a-51f9-4d4f-9f77-b836644e1c54, pool_id: 29e61161-eea7-4c3e-8e3e-8d8a030f4b7b (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.921193623Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/266edb61-c6f9-4597-9662-11a7fcae56f7/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.921200143Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c347173a-b2f9-4fe0-9e29-72f0b24d1c5f/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.921206133Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.921211573Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:681"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.921216313Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.921221213Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.921227154Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:24.921231374Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:25.057878943Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:25.057916674Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"af4109c5-bbac-4d47-a0e2-4b1267a1cb61 (zpool)"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:25.474131891Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.149818805s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/574b7f2f-83a4-499c-82e5-b830eda888e0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-20T16:05:25.738179872Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:26.146844812Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: af4109c5-bbac-4d47-a0e2-4b1267a1cb61 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:26.350355062Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:26.350478355Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:26.350488896Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:26.350504956Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:26.350523047Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:26.350665971Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:26.350678422Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:26.768414986Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/574b7f2f-83a4-499c-82e5-b830eda888e0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:27.235095908Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:27.23517101Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.323965418s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/574b7f2f-83a4-499c-82e5-b830eda888e0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-20T16:05:27.235319945Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:27.457802517Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 56.555415ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:27.654760432Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Automatically destroying dataset: oxp_af4109c5-bbac-4d47-a0e2-4b1267a1cb61/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:27.874952233Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 60.502841ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:28.094116571Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 190.930155ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:28.753222008Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 377.055291ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:29.156645403Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/574b7f2f-83a4-499c-82e5-b830eda888e0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-20T16:05:29.156695245Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/574b7f2f-83a4-499c-82e5-b830eda888e0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:29.15686456Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 963.649621ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:29.950070077Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:29.95175143Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.606629979s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/574b7f2f-83a4-499c-82e5-b830eda888e0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-20T16:05:30.162059581Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_af4109c5-bbac-4d47-a0e2-4b1267a1cb61","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:30.606405847Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/574b7f2f-83a4-499c-82e5-b830eda888e0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:30.792555764Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:30.798383356Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:30.798395646Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:30.798405977Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:30.798410397Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:30.798414227Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:30.996392112Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:30.996414883Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"6299fefa-5c12-4270-b629-f723a7cb7885 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:32.067349879Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 6299fefa-5c12-4270-b629-f723a7cb7885 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:32.313001485Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:32.313061767Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:32.313070737Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:32.313082678Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:32.313090038Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:32.313213962Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:32.313226222Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:32.752421087Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"10.561726919s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/574b7f2f-83a4-499c-82e5-b830eda888e0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-20T16:05:32.752652584Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/574b7f2f-83a4-499c-82e5-b830eda888e0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:33.421890346Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:33.647497856Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/574b7f2f-83a4-499c-82e5-b830eda888e0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-20T16:05:34.127905598Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Automatically destroying dataset: oxp_6299fefa-5c12-4270-b629-f723a7cb7885/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:34.128091724Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.03074942Z","hostname":"buskin","pid":1562,"component":"VmmReservoirManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/574b7f2f-83a4-499c-82e5-b830eda888e0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-20T16:05:35.132138799Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_6299fefa-5c12-4270-b629-f723a7cb7885","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.141440789Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.149687877Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.149743959Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.149755129Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.149761619Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.149766819Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.159112202Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.159149023Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"8063aaf6-9dd7-4b2a-b096-66bd421c61e1 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.253855832Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: 8063aaf6-9dd7-4b2a-b096-66bd421c61e1 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.26147562Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.261507661Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.261515332Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.261521012Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.261532972Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.261685687Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.261693937Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.297180326Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/574b7f2f-83a4-499c-82e5-b830eda888e0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:35.30113836Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Automatically destroying dataset: oxp_8063aaf6-9dd7-4b2a-b096-66bd421c61e1/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.305096073Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_8063aaf6-9dd7-4b2a-b096-66bd421c61e1","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.42306259Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.429965875Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.430010947Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.430018907Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.430024647Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.430032097Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.439871335Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.439893726Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"0fd8bd93-1766-40ff-af7d-07d69a799d88 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.534183792Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 0fd8bd93-1766-40ff-af7d-07d69a799d88 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.542371278Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.542394759Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.542401329Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.54242465Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.54243645Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.542563204Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.542570594Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.578388904Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_0fd8bd93-1766-40ff-af7d-07d69a799d88/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.586462106Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_0fd8bd93-1766-40ff-af7d-07d69a799d88","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.711332338Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.719751831Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.719786592Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.719794002Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.719803303Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.719809933Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.729973681Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.729998351Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"c02d0d46-c9bc-47f2-991b-6cec4937c8f3 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.824643139Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: c02d0d46-c9bc-47f2-991b-6cec4937c8f3 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.832260107Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.832284487Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.832292038Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.832336389Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.832388011Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.832611638Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.832618388Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.867389975Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_c02d0d46-c9bc-47f2-991b-6cec4937c8f3/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.875119286Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_c02d0d46-c9bc-47f2-991b-6cec4937c8f3","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:35.996199139Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.005705716Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.005736317Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.005780108Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.005803899Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.005808769Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.015545823Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.015573164Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"b2297356-8cd5-4606-999a-8889c6613486 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.109505689Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: b2297356-8cd5-4606-999a-8889c6613486 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.117584932Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.117610583Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.117618313Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.117630283Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.117640994Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.117783718Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.117790578Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/574b7f2f-83a4-499c-82e5-b830eda888e0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-20T16:05:36.134515381Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.152976838Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_b2297356-8cd5-4606-999a-8889c6613486/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.160845374Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_b2297356-8cd5-4606-999a-8889c6613486","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.282391252Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.290428143Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.290489685Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.290499525Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.290507056Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.290513416Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.300040754Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.300079605Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"459e4cea-0570-4a0f-b58a-37b154241edb (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.386540826Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 459e4cea-0570-4a0f-b58a-37b154241edb (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.393267487Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.393297288Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.393322458Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.393425782Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.393435082Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.393518364Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.393531905Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.40682272Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2303"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.429650504Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_459e4cea-0570-4a0f-b58a-37b154241edb/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.438449859Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_459e4cea-0570-4a0f-b58a-37b154241edb","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.567978456Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.576382889Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.5764163Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.57642494Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.57643033Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.576444761Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.585746971Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.585777943Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"0c51412c-3c0e-4ec7-9e61-170beed202ad (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.68075135Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 0c51412c-3c0e-4ec7-9e61-170beed202ad (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.688637537Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.68873967Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.68875014Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.688780991Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.688791482Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.688956397Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.688965367Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.725803098Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_0c51412c-3c0e-4ec7-9e61-170beed202ad/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.733437617Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_0c51412c-3c0e-4ec7-9e61-170beed202ad","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.855866072Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.863848122Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.863883233Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.863889443Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.863894683Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.863900923Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.873502023Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.873530355Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"29e61161-eea7-4c3e-8e3e-8d8a030f4b7b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.967226322Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 29e61161-eea7-4c3e-8e3e-8d8a030f4b7b (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.975836481Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.975868182Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.975874642Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.975916184Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.975935314Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.976073419Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:36.976080029Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.012056513Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_29e61161-eea7-4c3e-8e3e-8d8a030f4b7b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.021591521Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/574b7f2f-83a4-499c-82e5-b830eda888e0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-20T16:05:37.137572865Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_29e61161-eea7-4c3e-8e3e-8d8a030f4b7b","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.14413664Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.152273775Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.152304616Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/266edb61-c6f9-4597-9662-11a7fcae56f7/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.152365948Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.152508932Z","hostname":"buskin","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 266edb61-c6f9-4597-9662-11a7fcae56f7 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c347173a-b2f9-4fe0-9e29-72f0b24d1c5f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: af4109c5-bbac-4d47-a0e2-4b1267a1cb61 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: 6299fefa-5c12-4270-b629-f723a7cb7885 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: 8063aaf6-9dd7-4b2a-b096-66bd421c61e1 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: 0fd8bd93-1766-40ff-af7d-07d69a799d88 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: c02d0d46-c9bc-47f2-991b-6cec4937c8f3 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: b2297356-8cd5-4606-999a-8889c6613486 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: 459e4cea-0570-4a0f-b58a-37b154241edb (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: 0c51412c-3c0e-4ec7-9e61-170beed202ad (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: 29e61161-eea7-4c3e-8e3e-8d8a030f4b7b (zpool), kind: External } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.152568884Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.152576404Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Writing ledger to /pool/int/c347173a-b2f9-4fe0-9e29-72f0b24d1c5f/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.152667957Z","hostname":"buskin","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.152967936Z","hostname":"buskin","pid":1562,"uri":"/omicron-physical-disks","method":"PUT","req_id":"523c4a68-14c4-4402-8ef1-c7b1108b9f7d","remote_addr":"[fd00:1122:3344:101::1]:62347","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":12231929,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 63ca4438-daea-417e-9e0c-3d068d6765b2, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af4109c5-bbac-4d47-a0e2-4b1267a1cb61 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2860d1fc-15c4-44ef-a5b4-38c81ebfce6b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6299fefa-5c12-4270-b629-f723a7cb7885 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fdb08d15-c755-44f9-963c-113e9a0473d9, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8063aaf6-9dd7-4b2a-b096-66bd421c61e1 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.183503101Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.183666446Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:44658"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63ca4438-daea-417e-9e0c-3d068d6765b2, dataset_name: DatasetName { pool_name: ZpoolName { id: af4109c5-bbac-4d47-a0e2-4b1267a1cb61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.183966695Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.183982596Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63ca4438-daea-417e-9e0c-3d068d6765b2, dataset_name: DatasetName { pool_name: ZpoolName { id: af4109c5-bbac-4d47-a0e2-4b1267a1cb61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2860d1fc-15c4-44ef-a5b4-38c81ebfce6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 6299fefa-5c12-4270-b629-f723a7cb7885 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.235772654Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.235809785Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2860d1fc-15c4-44ef-a5b4-38c81ebfce6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 6299fefa-5c12-4270-b629-f723a7cb7885 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdb08d15-c755-44f9-963c-113e9a0473d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 8063aaf6-9dd7-4b2a-b096-66bd421c61e1 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.285526269Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.28556762Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fdb08d15-c755-44f9-963c-113e9a0473d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 8063aaf6-9dd7-4b2a-b096-66bd421c61e1 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/574b7f2f-83a4-499c-82e5-b830eda888e0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:37.298665919Z","hostname":"buskin","pid":1562,"component":"NexusNotifierTask","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.334373375Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/266edb61-c6f9-4597-9662-11a7fcae56f7/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.334417006Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c347173a-b2f9-4fe0-9e29-72f0b24d1c5f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.334429157Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.334483819Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.376721508Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.3767617Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.37677Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_2860d1fc-15c4-44ef-a5b4-38c81ebfce6b","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.430681795Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_fdb08d15-c755-44f9-963c-113e9a0473d9","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.434665149Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_63ca4438-daea-417e-9e0c-3d068d6765b2","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.438339134Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_internal_dns_2860d1fc-15c4-44ef-a5b4-38c81ebfce6b","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.45036245Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_fdb08d15-c755-44f9-963c-113e9a0473d9","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.455131239Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_63ca4438-daea-417e-9e0c-3d068d6765b2","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:37.459362191Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/574b7f2f-83a4-499c-82e5-b830eda888e0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-20T16:05:38.140018451Z","hostname":"buskin","pid":1562,"component":"ProbeManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Profile for oxz_internal_dns_2860d1fc-15c4-44ef-a5b4-38c81ebfce6b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:38.947190924Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:38.947285467Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:38.947483653Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_2860d1fc-15c4-44ef-a5b4-38c81ebfce6b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_internal_dns_fdb08d15-c755-44f9-963c-113e9a0473d9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:39.435955496Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:39.916574655Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"Profile for oxz_internal_dns_63ca4438-daea-417e-9e0c-3d068d6765b2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:39.916657768Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:39.916676789Z","hostname":"buskin","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"wait for service Some(\"oxz_internal_dns_2860d1fc-15c4-44ef-a5b4-38c81ebfce6b\") failed: Property not found. retry in 28.262181ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:39.924246975Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_2860d1fc-15c4-44ef-a5b4-38c81ebfce6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:39.924287446Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_fdb08d15-c755-44f9-963c-113e9a0473d9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:39.927997232Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_63ca4438-daea-417e-9e0c-3d068d6765b2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_2860d1fc-15c4-44ef-a5b4-38c81ebfce6b\") failed: Property not found. retry in 57.761364ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:39.962664426Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_2860d1fc-15c4-44ef-a5b4-38c81ebfce6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2860d1fc-15c4-44ef-a5b4-38c81ebfce6b\") failed: Property not found. retry in 129.689667ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:40.031792475Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_2860d1fc-15c4-44ef-a5b4-38c81ebfce6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fdb08d15-c755-44f9-963c-113e9a0473d9\") failed: Property not found. retry in 36.866387ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:40.057264811Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_fdb08d15-c755-44f9-963c-113e9a0473d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_63ca4438-daea-417e-9e0c-3d068d6765b2\") failed: Property not found. retry in 63.108571ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:40.06300993Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_63ca4438-daea-417e-9e0c-3d068d6765b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fdb08d15-c755-44f9-963c-113e9a0473d9\") failed: Property not found. retry in 50.985903ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:40.101182873Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_fdb08d15-c755-44f9-963c-113e9a0473d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_63ca4438-daea-417e-9e0c-3d068d6765b2\") failed: Property not found. retry in 53.023429ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:40.187224982Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_63ca4438-daea-417e-9e0c-3d068d6765b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2860d1fc-15c4-44ef-a5b4-38c81ebfce6b\") failed: Property not found. retry in 332.027757ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:40.260738279Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_2860d1fc-15c4-44ef-a5b4-38c81ebfce6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fdb08d15-c755-44f9-963c-113e9a0473d9\") failed: Property not found. retry in 224.002822ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:40.272319261Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_fdb08d15-c755-44f9-963c-113e9a0473d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_63ca4438-daea-417e-9e0c-3d068d6765b2\") failed: Property not found. retry in 227.902584ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:40.314043075Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_63ca4438-daea-417e-9e0c-3d068d6765b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fdb08d15-c755-44f9-963c-113e9a0473d9\") failed: Property not found. retry in 217.357752ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:40.514312643Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_fdb08d15-c755-44f9-963c-113e9a0473d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_63ca4438-daea-417e-9e0c-3d068d6765b2\") failed: Property not found. retry in 374.650785ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:40.551182945Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_63ca4438-daea-417e-9e0c-3d068d6765b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2860d1fc-15c4-44ef-a5b4-38c81ebfce6b\") failed: Property not found. retry in 809.556002ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:40.602420226Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_2860d1fc-15c4-44ef-a5b4-38c81ebfce6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fdb08d15-c755-44f9-963c-113e9a0473d9\") failed: Property not found. retry in 1.082712835s","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:40.740932125Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_fdb08d15-c755-44f9-963c-113e9a0473d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_63ca4438-daea-417e-9e0c-3d068d6765b2\") failed: Property not found. retry in 698.912087ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:40.935547836Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_63ca4438-daea-417e-9e0c-3d068d6765b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2860d1fc-15c4-44ef-a5b4-38c81ebfce6b\") failed: Property not found. retry in 505.990752ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:41.422495462Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_2860d1fc-15c4-44ef-a5b4-38c81ebfce6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_63ca4438-daea-417e-9e0c-3d068d6765b2\") failed: Property not found. retry in 999.362012ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:41.643946682Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_63ca4438-daea-417e-9e0c-3d068d6765b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fdb08d15-c755-44f9-963c-113e9a0473d9\") failed: Property not found. retry in 1.096110327s","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:41.835816508Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_fdb08d15-c755-44f9-963c-113e9a0473d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2860d1fc-15c4-44ef-a5b4-38c81ebfce6b\") failed: Property not found. retry in 652.780302ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:41.937820615Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_2860d1fc-15c4-44ef-a5b4-38c81ebfce6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:9fb2::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:42.47912128Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2320"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:42.479144251Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"wait for service Some(\"oxz_internal_dns_2860d1fc-15c4-44ef-a5b4-38c81ebfce6b\") failed: Property not found. retry in 1.214568628s","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:42.599704309Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_2860d1fc-15c4-44ef-a5b4-38c81ebfce6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_63ca4438-daea-417e-9e0c-3d068d6765b2\") failed: Property not found. retry in 1.083249169s","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:42.650459704Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_63ca4438-daea-417e-9e0c-3d068d6765b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fdb08d15-c755-44f9-963c-113e9a0473d9\") failed: Property not found. retry in 727.036988ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:42.939180087Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_fdb08d15-c755-44f9-963c-113e9a0473d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fef5:b724","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:43.024726599Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2327"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:43.030621773Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2388"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:43.14923782Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2573"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:43.391082287Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2737"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:43.577283646Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2452"} {"msg":"wait for service Some(\"oxz_internal_dns_fdb08d15-c755-44f9-963c-113e9a0473d9\") failed: Property not found. retry in 1.183392265s","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:43.676105604Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_fdb08d15-c755-44f9-963c-113e9a0473d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_63ca4438-daea-417e-9e0c-3d068d6765b2\") failed: Property not found. retry in 1.044022043s","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:43.743454478Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_63ca4438-daea-417e-9e0c-3d068d6765b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2860d1fc-15c4-44ef-a5b4-38c81ebfce6b\") failed: Property not found. retry in 926.739328ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:43.821805437Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_2860d1fc-15c4-44ef-a5b4-38c81ebfce6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:43.884485416Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2487"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:44.097876904Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2787"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:44.276584608Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2814"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:44.491643709Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2681"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:44.674539094Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2481"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:44.685546748Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3873","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:44.685588359Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3889"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:44.685598319Z","hostname":"buskin","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"wait for service Some(\"oxz_internal_dns_63ca4438-daea-417e-9e0c-3d068d6765b2\") failed: Property not found. retry in 1.28022264s","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:44.798839778Z","hostname":"buskin","pid":1562,"zone":"oxz_internal_dns_63ca4438-daea-417e-9e0c-3d068d6765b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.239205058Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3897"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.245365611Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3953"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.384293082Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4015"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.484839224Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3996","rack_subnet":"fd00:1122:3344::"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.506973666Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4043"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.604250396Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4070"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.725763253Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:365"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.758101013Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:413"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.789689751Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:651"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.789773613Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:689"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.790154415Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:692","uptime":"2"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.790173736Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:435","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.864771207Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.8648651Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.86487547Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.86488412Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.864891981Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.864900031Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.864905911Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.864913481Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.864922582Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.864946332Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.864952942Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.864959272Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.864963633Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.864969663Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.864973913Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.864979993Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.864983973Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.864989644Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.864993734Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.865000094Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.865004504Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.865014904Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:45.865019205Z","hostname":"buskin","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.097513099Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_2860d1fc-15c4-44ef-a5b4-38c81ebfce6b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.097568091Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_fdb08d15-c755-44f9-963c-113e9a0473d9"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.097576581Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_63ca4438-daea-417e-9e0c-3d068d6765b2"} {"msg":"Writing ledger to /pool/int/266edb61-c6f9-4597-9662-11a7fcae56f7/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.097589371Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c347173a-b2f9-4fe0-9e29-72f0b24d1c5f/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.097928042Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.098250362Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"672d62ce-ec4e-4569-bc44-5604149a36cd","remote_addr":"[fd00:1122:3344:101::1]:44658","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8914496,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.099901603Z","hostname":"buskin","pid":1562,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:548"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.14522429Z","hostname":"buskin","pid":1562,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:548"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.198157134Z","hostname":"buskin","pid":1562,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:548"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.24253729Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:584"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.242550361Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.273937032Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.274651294Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.274682995Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 63ca4438-daea-417e-9e0c-3d068d6765b2, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af4109c5-bbac-4d47-a0e2-4b1267a1cb61 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2860d1fc-15c4-44ef-a5b4-38c81ebfce6b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6299fefa-5c12-4270-b629-f723a7cb7885 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fdb08d15-c755-44f9-963c-113e9a0473d9, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8063aaf6-9dd7-4b2a-b096-66bd421c61e1 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bae356d5-7113-4cd2-8cd4-c89fd67c9f2d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2b178171-a0b1-4309-824f-7ebd0cbcbbb2, kind: Service { id: bae356d5-7113-4cd2-8cd4-c89fd67c9f2d }, name: Name(\"ntp-bae356d5-7113-4cd2-8cd4-c89fd67c9f2d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 108])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.302822585Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.302937018Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:63290"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63ca4438-daea-417e-9e0c-3d068d6765b2, dataset_name: DatasetName { pool_name: ZpoolName { id: af4109c5-bbac-4d47-a0e2-4b1267a1cb61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.303152465Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.303162235Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63ca4438-daea-417e-9e0c-3d068d6765b2, dataset_name: DatasetName { pool_name: ZpoolName { id: af4109c5-bbac-4d47-a0e2-4b1267a1cb61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2860d1fc-15c4-44ef-a5b4-38c81ebfce6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 6299fefa-5c12-4270-b629-f723a7cb7885 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.324666387Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.324732749Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2860d1fc-15c4-44ef-a5b4-38c81ebfce6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 6299fefa-5c12-4270-b629-f723a7cb7885 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdb08d15-c755-44f9-963c-113e9a0473d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 8063aaf6-9dd7-4b2a-b096-66bd421c61e1 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.343572278Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.343610869Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fdb08d15-c755-44f9-963c-113e9a0473d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 8063aaf6-9dd7-4b2a-b096-66bd421c61e1 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.36219378Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/266edb61-c6f9-4597-9662-11a7fcae56f7/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.362251431Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c347173a-b2f9-4fe0-9e29-72f0b24d1c5f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.362390926Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.362493909Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.373587036Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.405572395Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.406213555Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.406236826Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.450634593Z","hostname":"buskin","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 255, 108]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.450670365Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"ntp"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.45180251Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.45181781Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.45211465Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.453116151Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_ntp_bae356d5-7113-4cd2-8cd4-c89fd67c9f2d","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.476232823Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_bae356d5-7113-4cd2-8cd4-c89fd67c9f2d","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:46.495540127Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_bae356d5-7113-4cd2-8cd4-c89fd67c9f2d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:47.119243056Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:47.11938956Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_bae356d5-7113-4cd2-8cd4-c89fd67c9f2d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_bae356d5-7113-4cd2-8cd4-c89fd67c9f2d\") failed: Property not found. retry in 27.466131ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:47.252348885Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_bae356d5-7113-4cd2-8cd4-c89fd67c9f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_bae356d5-7113-4cd2-8cd4-c89fd67c9f2d\") failed: Property not found. retry in 129.558544ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:47.287670089Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_bae356d5-7113-4cd2-8cd4-c89fd67c9f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_bae356d5-7113-4cd2-8cd4-c89fd67c9f2d\") failed: Property not found. retry in 156.947589ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:47.451077445Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_bae356d5-7113-4cd2-8cd4-c89fd67c9f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_bae356d5-7113-4cd2-8cd4-c89fd67c9f2d\") failed: Property not found. retry in 448.972738ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:47.619251601Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_bae356d5-7113-4cd2-8cd4-c89fd67c9f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_bae356d5-7113-4cd2-8cd4-c89fd67c9f2d\") failed: Property not found. retry in 1.041632729s","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:48.077154858Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_bae356d5-7113-4cd2-8cd4-c89fd67c9f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_bae356d5-7113-4cd2-8cd4-c89fd67c9f2d\") failed: Property not found. retry in 836.329821ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:49.128521743Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_bae356d5-7113-4cd2-8cd4-c89fd67c9f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:49.171590998Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54795"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:49.171796145Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"b625ba6f-c53c-4c42-a2f6-8cff0196e8e9","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54795","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":83,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_bae356d5-7113-4cd2-8cd4-c89fd67c9f2d\") failed: Property not found. retry in 1.139617527s","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:49.973994512Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_bae356d5-7113-4cd2-8cd4-c89fd67c9f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_bae356d5-7113-4cd2-8cd4-c89fd67c9f2d\") failed: Property not found. retry in 944.056415ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:51.124101431Z","hostname":"buskin","pid":1562,"zone":"oxz_ntp_bae356d5-7113-4cd2-8cd4-c89fd67c9f2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:52.089377255Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_ntp_bae356d5-7113-4cd2-8cd4-c89fd67c9f2d"} {"msg":"Writing ledger to /pool/int/266edb61-c6f9-4597-9662-11a7fcae56f7/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:52.089441657Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c347173a-b2f9-4fe0-9e29-72f0b24d1c5f/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:52.089757867Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:52.090079367Z","hostname":"buskin","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"01b702ff-6bef-409d-a922-d4cab0d363ed","remote_addr":"[fd00:1122:3344:101::1]:63290","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":5787075,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:52.092235525Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:625"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:52.121663094Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:52.12183656Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:60949"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_bae356d5-7113-4cd2-8cd4-c89fd67c9f2d': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:52.134919618Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:52.135043572Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"e430d84d-85c4-495b-aa66-d7f8f311b002","remote_addr":"[fd00:1122:3344:101::1]:60949","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":13082,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:52.137679055Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:52.279773045Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:52.27993189Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:57247"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_bae356d5-7113-4cd2-8cd4-c89fd67c9f2d': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:56.288972737Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:56.289140462Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"9b9eff6d-7447-4e9a-bcc1-7df41576f664","remote_addr":"[fd00:1122:3344:101::1]:57247","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009092,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:56.29261812Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:56.418961988Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:56.419102733Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:58463"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_bae356d5-7113-4cd2-8cd4-c89fd67c9f2d': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:56.428124655Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:56.428228318Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"db96e3bc-2d41-44ad-9b4c-c2ff7670a401","remote_addr":"[fd00:1122:3344:101::1]:58463","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8998,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:05:56.430908502Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:56.599151559Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:05:56.599270223Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:41609"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_bae356d5-7113-4cd2-8cd4-c89fd67c9f2d': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:00.608591658Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:00.608796955Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"4a2ca3f8-3406-432c-af93-a93a9119e81c","remote_addr":"[fd00:1122:3344:101::1]:41609","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009393,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:00.611109947Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:00.886324677Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:00.886464402Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:53605"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:00.894988088Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"e353c1db-7790-459a-b3f5-658b3b23cd68","remote_addr":"[fd00:1122:3344:101::1]:53605","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8422,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:00.898755895Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:00.898763816Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:01.041834336Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:01.041922248Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:56120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:01.05157432Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"1164b567-5639-4505-abf3-aa34a4ee7cca","remote_addr":"[fd00:1122:3344:101::1]:56120","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":9565,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:01.053618344Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:01.053631445Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:01.452716865Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:01.45284586Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:41762"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:01.461789699Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"73194e7f-ce68-4cde-97c1-b6116de24aeb","remote_addr":"[fd00:1122:3344:101::1]:41762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8820,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:01.464617547Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:01.464629768Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:01.845866201Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:01.845983165Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:45643"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:01.854568153Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"f4ae159e-ee6d-4ec2-ab5e-ab054b3c86d0","remote_addr":"[fd00:1122:3344:101::1]:45643","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8441,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:01.857163564Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:01.857168844Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:02.368276115Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:02.368382768Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:49836"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:02.376339457Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"51994fec-86ba-4e51-9888-17cb2acda9ab","remote_addr":"[fd00:1122:3344:101::1]:49836","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":7817,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:02.378652799Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:02.378659759Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:03.016945755Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:03.017053178Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:35829"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:03.025165462Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"a684988e-0fdb-4196-a690-c7ccd2f4c32c","remote_addr":"[fd00:1122:3344:101::1]:35829","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8037,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:03.027273028Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:03.027277208Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:03.368675696Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:03.368840581Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:55534"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:03.374855159Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"338.217188ms"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:03.377791461Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"68b352af-5eee-489c-b4c0-07736456b6fc","remote_addr":"[fd00:1122:3344:101::1]:55534","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8898,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:03.380249288Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:03.380255888Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:03.149722409Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:03.149823362Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:46955"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:03.157871063Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"98d6ab24-6ca2-43e7-8a47-650d9a3f6335","remote_addr":"[fd00:1122:3344:101::1]:46955","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":7957,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:03.159527935Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:03.159533585Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:04.21122459Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:04.211324943Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:57457"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:04.220531751Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"3344e4b4-8608-493f-aad7-3e05980f3dd4","remote_addr":"[fd00:1122:3344:101::1]:57457","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":9117,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:04.224437303Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:04.224446303Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:05.377307378Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:05.377402901Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:44415"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:05.385460523Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"4bab3a73-b5e2-44bf-90d3-7b7610bf3876","remote_addr":"[fd00:1122:3344:101::1]:44415","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":7980,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:05.386785354Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:05.386790245Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:07.527240795Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:07.527371939Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:49698"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:07.535987179Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"40821f72-73d5-461d-bf6c-6500671eac5f","remote_addr":"[fd00:1122:3344:101::1]:49698","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8459,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:07.538843638Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:07.538850078Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:08.414245353Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:08.414364137Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:61703"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:08.422979906Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"c56afffb-846c-43f8-b2c6-f8e77177d563","remote_addr":"[fd00:1122:3344:101::1]:61703","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8471,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:08.425441833Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:08.425450303Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:09.610561446Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:09.6106806Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:52299"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:09.619803905Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"ad67dbd5-ec92-4637-8ee0-81b92afb6303","remote_addr":"[fd00:1122:3344:101::1]:52299","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":9013,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:09.62124303Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:09.62124935Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:11.472588732Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:11.472748386Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:59161"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:11.484150813Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"d88c8b07-300a-4e2f-aa57-5b5b6aa02e7d","remote_addr":"[fd00:1122:3344:101::1]:59161","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":11319,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:11.487358333Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:11.487368433Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:13.496439094Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:13.496563638Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:39383"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:13.505153786Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"5383b446-9795-48ca-b37e-f101806b62e2","remote_addr":"[fd00:1122:3344:101::1]:39383","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8438,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:13.507400716Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:13.507408497Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:17.498543703Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:17.498665246Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:62195"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:17.507943916Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"eca30565-f328-414c-9ad1-6781bbd35f02","remote_addr":"[fd00:1122:3344:101::1]:62195","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":9094,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:17.510273319Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:17.51028054Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:18.901709059Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"326.410821ms"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:19.358781291Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48567"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:19.359531205Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"ee973e63-831c-46cb-94a6-0bd08906f1ee","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48567","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":710,"response_code":"200"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:21.283657132Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:21.283795086Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:37099"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:21.293117147Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"a2157719-419e-43e4-b840-0b2b106d2ea7","remote_addr":"[fd00:1122:3344:101::1]:37099","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":9190,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:21.295317226Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:21.295325386Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:24.673835549Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:24.673972004Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:55493"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:24.682753318Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"77d7b45f-1fa4-4eca-8d2d-2f9b07c2b276","remote_addr":"[fd00:1122:3344:101::1]:55493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8660,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:24.686314449Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:24.686320629Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:31.515194572Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:31.515368917Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:60519"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:31.526291139Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"8e47c095-4a18-4caa-aa56-777b47d91a9d","remote_addr":"[fd00:1122:3344:101::1]:60519","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":10829,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:31.530587293Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:31.530594233Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:34.261085325Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"824.736531ms"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:35.374203609Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:35.374308642Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:39135"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:35.382778477Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"6869e2da-cbce-4093-b4dc-98f55fa4fbc2","remote_addr":"[fd00:1122:3344:101::1]:39135","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8351,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:35.385068679Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:35.38510152Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:44.240033592Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:44.240158226Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:42827"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:44.249060054Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"3bda0ce9-d577-473c-9a06-e8dcef96402e","remote_addr":"[fd00:1122:3344:101::1]:42827","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8648,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:44.251408838Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:44.251417838Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:49.393770817Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38240"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:49.39385016Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"41f9d2b2-df2f-47e2-ab05-75f31116827b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38240","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":33,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:50.120204348Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"1.859560282s"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:55.135895359Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:55.136025813Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:49277"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:55.146358606Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"17dca93a-8975-4f29-9cda-af3a37c139a9","remote_addr":"[fd00:1122:3344:101::1]:49277","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":10167,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:06:55.14873655Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:06:55.148743241Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:07:07.014273789Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"3.410423947s"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:07:07.755332126Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:07:07.75546753Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:59120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:07:07.763978606Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"2d66d023-a1f2-425f-b32b-9253d04cf300","remote_addr":"[fd00:1122:3344:101::1]:59120","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8398,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:07:07.766407102Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:07:07.766416822Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:07:18.063197362Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:07:18.063314336Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:45830"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:07:18.071906604Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"730de18e-72a1-4222-8e08-8110bc237ac7","remote_addr":"[fd00:1122:3344:101::1]:45830","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8351,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2554765158, ip_addr: 152.70.159.102, stratum: 3, ref_time: 1716221227.7827995, correction: -0.385826111 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:07:18.074675131Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:07:18.074689342Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:07:19.424766935Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65310"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:07:19.424850927Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"6ab6b42b-192f-43a4-80ab-ca75c313f33c","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65310","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":46,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:07:25.456114914Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"11.110003904s"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:07:32.392203078Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:07:32.392322811Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:56044"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:07:32.401517479Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"90a55d7b-b244-447a-bbc2-85ec740ae158","remote_addr":"[fd00:1122:3344:101::1]:56044","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":9069,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2554765158, ip_addr: 152.70.159.102, stratum: 3, ref_time: 1716221227.7827995, correction: -0.346911252 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:07:32.40539907Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:07:32.405422631Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:07:40.832479764Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:07:40.832613698Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:33435"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:07:40.841411343Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"505bb9be-807e-4254-931e-3f01036e0ecf","remote_addr":"[fd00:1122:3344:101::1]:33435","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8688,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2554765158, ip_addr: 152.70.159.102, stratum: 3, ref_time: 1716221227.7827995, correction: -0.323990971 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:07:40.843980293Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:07:40.843990203Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:07:49.453989132Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63965"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:07:49.454060394Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"bcff466f-df06-45e0-83bc-214e331e4d2e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63965","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":38,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:07:51.599338872Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"9.236437833s"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:07:57.139077495Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:07:57.139199558Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:42075"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:07:57.148456058Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"9c5e9dd3-1464-42c4-9a76-c6f457ecee0c","remote_addr":"[fd00:1122:3344:101::1]:42075","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":9081,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2554765158, ip_addr: 152.70.159.102, stratum: 3, ref_time: 1716221227.7827995, correction: -0.27970624 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:07:57.151205614Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:07:57.151212214Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:08:15.862115458Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"25.523843182s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:08:19.4707658Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62314"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:08:19.470832712Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"ab7be848-8c16-47e3-898e-0d938dfffb2e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62314","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":35,"response_code":"200"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:08:25.773201199Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:08:25.773329703Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:45675"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:08:25.782313194Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"cd722788-b973-413b-816e-8601d56a5626","remote_addr":"[fd00:1122:3344:101::1]:45675","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8841,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2554765158, ip_addr: 152.70.159.102, stratum: 3, ref_time: 1716221293.1075077, correction: -0.366916656 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:08:25.784966197Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:08:25.784987178Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:08:49.432772201Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60032"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:08:49.432824072Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"d9ccc9d2-955d-4f17-9f2e-8ea28f42958b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60032","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":43,"response_code":"200"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:08:49.513386456Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:08:49.513503409Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:43072"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:08:49.522545663Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"554d8a8b-001f-41a0-ab24-38f07f2719e9","remote_addr":"[fd00:1122:3344:101::1]:43072","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8920,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2554765158, ip_addr: 152.70.159.102, stratum: 3, ref_time: 1716221293.1075077, correction: -0.30244574 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:08:49.525049961Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:08:49.525060711Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:08:56.323496808Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"95.720393994s"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:09:14.316873271Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:09:14.316981375Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:44670"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:09:14.326436291Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"1e0b0598-e3be-4453-b452-0c47dc6b302c","remote_addr":"[fd00:1122:3344:101::1]:44670","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":9289,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2554765158, ip_addr: 152.70.159.102, stratum: 3, ref_time: 1716221293.1075077, correction: -0.235086113 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:09:14.329042132Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:09:14.329051403Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:09:19.39349713Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58480"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:09:19.394202862Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"7e5f95e1-291e-4f3b-8c40-24701e7ac50e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58480","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":689,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:09:49.34604506Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37986"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:09:49.346089121Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"6dd511b4-649b-4f8a-b1c7-e36ab616f78d","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37986","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":33,"response_code":"200"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:09:58.588827817Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:09:58.58894542Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:35998"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:09:58.599170161Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"8bc9ac7a-244b-4adc-a4d4-e63e93afe75a","remote_addr":"[fd00:1122:3344:101::1]:35998","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":10044,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2554765158, ip_addr: 152.70.159.102, stratum: 3, ref_time: 1716221357.840268, correction: -0.138335094 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:09:58.602104063Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:09:58.602135584Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:10:19.297448344Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38167"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:10:19.297515386Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"ddb58c07-6789-40d7-a688-a8f39f8e4cb1","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38167","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":41,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:10:36.933054845Z","hostname":"buskin","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-20T16:10:46.792975004Z","hostname":"buskin","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"574b7f2f-83a4-499c-82e5-b830eda888e0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"150.763091053s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:10:49.262820205Z","hostname":"buskin","pid":1562,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56292"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:10:49.262897628Z","hostname":"buskin","pid":1562,"uri":"/baseboard","method":"GET","req_id":"41424110-6486-47e5-bfc1-23a4685c6596","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56292","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":42,"response_code":"200"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:03.677904278Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:03.678036132Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:52736"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:03.699523384Z","hostname":"buskin","pid":1562,"uri":"/timesync","method":"GET","req_id":"40b1973c-c6ad-44d2-aab8-f6a7895ee13e","remote_addr":"[fd00:1122:3344:101::1]:52736","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":21310,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 852310310, ip_addr: 50.205.57.38, stratum: 2, ref_time: 1716221441.2542422, correction: -0.000929071 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:03.701980691Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:03.701991521Z","hostname":"buskin","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1216"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 63ca4438-daea-417e-9e0c-3d068d6765b2, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af4109c5-bbac-4d47-a0e2-4b1267a1cb61 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2860d1fc-15c4-44ef-a5b4-38c81ebfce6b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6299fefa-5c12-4270-b629-f723a7cb7885 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fdb08d15-c755-44f9-963c-113e9a0473d9, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8063aaf6-9dd7-4b2a-b096-66bd421c61e1 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bae356d5-7113-4cd2-8cd4-c89fd67c9f2d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2b178171-a0b1-4309-824f-7ebd0cbcbbb2, kind: Service { id: bae356d5-7113-4cd2-8cd4-c89fd67c9f2d }, name: Name(\"ntp-bae356d5-7113-4cd2-8cd4-c89fd67c9f2d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 108])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e2f4b6e4-ab13-4702-af6c-ebe5a6b8cb69, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: af4109c5-bbac-4d47-a0e2-4b1267a1cb61 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e26b544-d1a1-4ed6-a17e-a07dd62aaf11, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6299fefa-5c12-4270-b629-f723a7cb7885 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37bc3ed9-c555-43f8-83dd-39f835e8db19, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8063aaf6-9dd7-4b2a-b096-66bd421c61e1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 56753366-80fb-4ec3-9937-08d4aae9f7e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0fd8bd93-1766-40ff-af7d-07d69a799d88 (zpool), kind: External } } } }, OmicronZoneConfig { id: f72709b9-7a1b-4242-b249-cdf4d600be46, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c02d0d46-c9bc-47f2-991b-6cec4937c8f3 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:03.730372649Z","hostname":"buskin","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:03.730491373Z","hostname":"buskin","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:60424"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 63ca4438-daea-417e-9e0c-3d068d6765b2, dataset_name: DatasetName { pool_name: ZpoolName { id: af4109c5-bbac-4d47-a0e2-4b1267a1cb61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:03.73135502Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:03.73136159Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 63ca4438-daea-417e-9e0c-3d068d6765b2, dataset_name: DatasetName { pool_name: ZpoolName { id: af4109c5-bbac-4d47-a0e2-4b1267a1cb61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2860d1fc-15c4-44ef-a5b4-38c81ebfce6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 6299fefa-5c12-4270-b629-f723a7cb7885 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:03.752235773Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:03.752289935Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2860d1fc-15c4-44ef-a5b4-38c81ebfce6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 6299fefa-5c12-4270-b629-f723a7cb7885 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdb08d15-c755-44f9-963c-113e9a0473d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 8063aaf6-9dd7-4b2a-b096-66bd421c61e1 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:03.7709956Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:03.771040351Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fdb08d15-c755-44f9-963c-113e9a0473d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 8063aaf6-9dd7-4b2a-b096-66bd421c61e1 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2f4b6e4-ab13-4702-af6c-ebe5a6b8cb69, dataset_name: DatasetName { pool_name: ZpoolName { id: af4109c5-bbac-4d47-a0e2-4b1267a1cb61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:03.788022352Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:03.788065124Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e2f4b6e4-ab13-4702-af6c-ebe5a6b8cb69, dataset_name: DatasetName { pool_name: ZpoolName { id: af4109c5-bbac-4d47-a0e2-4b1267a1cb61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e26b544-d1a1-4ed6-a17e-a07dd62aaf11, dataset_name: DatasetName { pool_name: ZpoolName { id: 6299fefa-5c12-4270-b629-f723a7cb7885 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:03.842001531Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:03.842051022Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e26b544-d1a1-4ed6-a17e-a07dd62aaf11, dataset_name: DatasetName { pool_name: ZpoolName { id: 6299fefa-5c12-4270-b629-f723a7cb7885 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 37bc3ed9-c555-43f8-83dd-39f835e8db19, dataset_name: DatasetName { pool_name: ZpoolName { id: 8063aaf6-9dd7-4b2a-b096-66bd421c61e1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:03.89889589Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:03.898944092Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 37bc3ed9-c555-43f8-83dd-39f835e8db19, dataset_name: DatasetName { pool_name: ZpoolName { id: 8063aaf6-9dd7-4b2a-b096-66bd421c61e1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56753366-80fb-4ec3-9937-08d4aae9f7e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0fd8bd93-1766-40ff-af7d-07d69a799d88 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:03.95452657Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:03.954570472Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56753366-80fb-4ec3-9937-08d4aae9f7e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 0fd8bd93-1766-40ff-af7d-07d69a799d88 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f72709b9-7a1b-4242-b249-cdf4d600be46, dataset_name: DatasetName { pool_name: ZpoolName { id: c02d0d46-c9bc-47f2-991b-6cec4937c8f3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:04.004004258Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:04.004051539Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f72709b9-7a1b-4242-b249-cdf4d600be46, dataset_name: DatasetName { pool_name: ZpoolName { id: c02d0d46-c9bc-47f2-991b-6cec4937c8f3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:04.062327662Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/266edb61-c6f9-4597-9662-11a7fcae56f7/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:04.062393794Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c347173a-b2f9-4fe0-9e29-72f0b24d1c5f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:04.062629881Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:04.062776186Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:04.147066193Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:04.147126204Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:04.147134365Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:04.147140435Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:04.147148335Z","hostname":"buskin","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_e2f4b6e4-ab13-4702-af6c-ebe5a6b8cb69","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:04.237899394Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_8e26b544-d1a1-4ed6-a17e-a07dd62aaf11","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:04.241893379Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_56753366-80fb-4ec3-9937-08d4aae9f7e9","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:04.245952336Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_f72709b9-7a1b-4242-b249-cdf4d600be46","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:04.250055534Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_37bc3ed9-c555-43f8-83dd-39f835e8db19","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:04.254643913Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_e2f4b6e4-ab13-4702-af6c-ebe5a6b8cb69","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:04.259373566Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_8e26b544-d1a1-4ed6-a17e-a07dd62aaf11","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:04.263888607Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_56753366-80fb-4ec3-9937-08d4aae9f7e9","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:04.268129099Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_f72709b9-7a1b-4242-b249-cdf4d600be46","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:04.272469405Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_37bc3ed9-c555-43f8-83dd-39f835e8db19","v":0,"name":"SledAgent","level":30,"time":"2024-05-20T16:11:04.277082729Z","hostname":"buskin","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}