[ May 24 15:58:28 Enabled. ] [ May 24 15:58:28 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ May 24 15:58:28 Method "start" exited with status 0. ] [ May 24 15:58:28 Rereading configuration. ] [ May 24 15:58:28 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:28.420474312Z","hostname":"sock","pid":1562,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:28.420641554Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:28.473458992Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:28.473509973Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:28.589213737Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:28.59806788Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:28.605939592Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.207473242Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fdb0:200::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.348611424Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:119"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.348650115Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:131"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.348747165Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.348846217Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:162","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.348855207Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:506"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.348898847Z","hostname":"sock","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:625"} {"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.352135395Z","hostname":"sock","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:29.352142415Z","hostname":"sock","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:674"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.352156256Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.352160585Z","hostname":"sock","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:733"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.352172936Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.352215946Z","hostname":"sock","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.352265107Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.352278787Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.352306677Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.363745921Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.363779301Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"07053fdc-4cea-499c-875c-c222ca87c041 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.461500175Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 07053fdc-4cea-499c-875c-c222ca87c041 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.666527595Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.666562525Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.666567195Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.666572145Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.666583405Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.666588085Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.666603616Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.675044134Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.675063455Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"02661193-169e-480c-af92-d631121af19a (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.765667605Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 02661193-169e-480c-af92-d631121af19a (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.970730755Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.970772345Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 07053fdc-4cea-499c-875c-c222ca87c041 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 02661193-169e-480c-af92-d631121af19a (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.970795555Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.970802695Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.970819876Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.970827186Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.970845886Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_1.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.970889206Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.970895406Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 07053fdc-4cea-499c-875c-c222ca87c041 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 02661193-169e-480c-af92-d631121af19a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.970910247Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.970915717Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.970920727Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.970925107Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_2.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.970934117Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.970938817Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.970944107Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 07053fdc-4cea-499c-875c-c222ca87c041 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 02661193-169e-480c-af92-d631121af19a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.970970987Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.970977898Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.970982058Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_3.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.970987018Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.970990818Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.970999888Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 07053fdc-4cea-499c-875c-c222ca87c041 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 02661193-169e-480c-af92-d631121af19a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971016088Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_4.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971021588Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971026318Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 07053fdc-4cea-499c-875c-c222ca87c041 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 02661193-169e-480c-af92-d631121af19a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971052878Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971058168Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971064428Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971068539Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971073419Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971077579Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_5.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971084069Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971088159Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971092899Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 07053fdc-4cea-499c-875c-c222ca87c041 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 02661193-169e-480c-af92-d631121af19a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971113589Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971119089Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971123759Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 07053fdc-4cea-499c-875c-c222ca87c041 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 02661193-169e-480c-af92-d631121af19a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971150469Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971156849Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.97116289Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.97116689Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.97117159Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.97117584Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_7.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.97118293Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 07053fdc-4cea-499c-875c-c222ca87c041 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 02661193-169e-480c-af92-d631121af19a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.97120183Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.97120799Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.97121264Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.97121942Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.97122397Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.9712286Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.97123965Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 07053fdc-4cea-499c-875c-c222ca87c041 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 02661193-169e-480c-af92-d631121af19a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971262231Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971266841Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971271671Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971275441Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:91"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971287631Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:93"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971291641Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 07053fdc-4cea-499c-875c-c222ca87c041 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 02661193-169e-480c-af92-d631121af19a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971323782Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971328791Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971332891Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971337362Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"No ledger in /pool/int/07053fdc-4cea-499c-875c-c222ca87c041/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971341762Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/02661193-169e-480c-af92-d631121af19a/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971350312Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971354182Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/07053fdc-4cea-499c-875c-c222ca87c041/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971376042Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/02661193-169e-480c-af92-d631121af19a/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971380542Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971384362Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971390892Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.971394212Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.982511602Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:711"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.982596014Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.982603514Z","hostname":"sock","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999922869s","next_cleanup":"Instant { tv_sec: 17328, tv_nsec: 608409320 }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.982656894Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/07053fdc-4cea-499c-875c-c222ca87c041/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.982718305Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/02661193-169e-480c-af92-d631121af19a/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.984673088Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.98574994Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.985828351Z","hostname":"sock","pid":1562,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:200::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV1 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV1 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n },\n ],\n addresses: [\n V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.986074734Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1003"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.986080644Z","hostname":"sock","pid":1562,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.986135755Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/07053fdc-4cea-499c-875c-c222ca87c041/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.986182315Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/02661193-169e-480c-af92-d631121af19a/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.986197616Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.986205366Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1063"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.986245476Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/07053fdc-4cea-499c-875c-c222ca87c041/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.986260626Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/02661193-169e-480c-af92-d631121af19a/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.986270377Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.986276926Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1104"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.986294557Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"} {"msg":"Creating plan for the sled at fdb0:200::1","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.986298367Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.986302487Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/07053fdc-4cea-499c-875c-c222ca87c041/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.986326197Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/02661193-169e-480c-af92-d631121af19a/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.986652911Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.986793983Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.986817973Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1137"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.986869044Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/07053fdc-4cea-499c-875c-c222ca87c041/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.986877693Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/02661193-169e-480c-af92-d631121af19a/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.987034695Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.987174467Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.987213928Z","hostname":"sock","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:200::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: f2da503c-9ac6-41a4-a9b4-f11c5d56071c, rack_id: acf2daca-68f5-4625-bcc1-ba20a57a103e, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.98740334Z","hostname":"sock","pid":1562,"remote_addr":"[fdb0:200::1]:42215","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: f2da503c-9ac6-41a4-a9b4-f11c5d56071c, rack_id: acf2daca-68f5-4625-bcc1-ba20a57a103e, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.987491271Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:360"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.987501751Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:375"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.987529021Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.987534341Z","hostname":"sock","pid":1562,"file":"sled-agent/src/server.rs:44"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.987538621Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:530"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.987544721Z","hostname":"sock","pid":1562,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.987550071Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"No ledger in /pool/int/07053fdc-4cea-499c-875c-c222ca87c041/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.987613712Z","hostname":"sock","pid":1562,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/02661193-169e-480c-af92-d631121af19a/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.987619742Z","hostname":"sock","pid":1562,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.987629582Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:519"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.987635292Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:547"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:29.990651428Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3849"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.000077858Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.010297838Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.019587536Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.028573561Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.037498926Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.045924075Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.054162711Z","hostname":"sock","pid":1562,"sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:353"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.054225832Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.054249952Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Failed enable ddm stats (will retry after 71.898648ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.056826532Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cb1d5921-ff2b-4993-8cc7-0f2443fd3532\", \"content-length\": \"124\", \"date\": \"Fri, 24 May 2024 15:58:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cb1d5921-ff2b-4993-8cc7-0f2443fd3532\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.062464898Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.070591883Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"No disks to be automatically managed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.070642024Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:558"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.070649644Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.070654894Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.070669254Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:370"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.070728015Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.070737025Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:449","result":"DisksManagementResult { status: [] }"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.070753035Z","hostname":"sock","pid":1562,"sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:365"} {"msg":"attempting to create encrypted zvol: name=\"oxi_07053fdc-4cea-499c-875c-c222ca87c041/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.083515324Z","hostname":"sock","pid":1562,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.096157672Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"successfully created encrypted zvol: name=\"oxi_07053fdc-4cea-499c-875c-c222ca87c041/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.107211012Z","hostname":"sock","pid":1562,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_07053fdc-4cea-499c-875c-c222ca87c041/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.107255102Z","hostname":"sock","pid":1562,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.111875896Z","hostname":"sock","pid":1562,"sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:380"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.111885167Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_07053fdc-4cea-499c-875c-c222ca87c041/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.111895147Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.114190194Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.129465002Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 114.854869ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.129972808Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c79d6b0d-0959-4bbd-ab8e-c8751bf6417d\", \"content-length\": \"124\", \"date\": \"Fri, 24 May 2024 15:58:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c79d6b0d-0959-4bbd-ab8e-c8751bf6417d\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.183358653Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.245683852Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 92.985765ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.24630086Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6ed6204a-9784-414b-83fa-51926dcb6749\", \"content-length\": \"124\", \"date\": \"Fri, 24 May 2024 15:58:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6ed6204a-9784-414b-83fa-51926dcb6749\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.34036427Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 107.476282ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.340913317Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6bc77ec2-b78e-4e97-b093-b1289a86c108\", \"content-length\": \"124\", \"date\": \"Fri, 24 May 2024 15:58:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6bc77ec2-b78e-4e97-b093-b1289a86c108\" }"} {"msg":"Mounting oxi_07053fdc-4cea-499c-875c-c222ca87c041/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.402955943Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.41642348Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.416508821Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.416533382Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.416545142Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.44969902Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 257.506967ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.450224966Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e3588af1-9ff1-4f1a-b300-a98c7e23209f\", \"content-length\": \"124\", \"date\": \"Fri, 24 May 2024 15:58:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e3588af1-9ff1-4f1a-b300-a98c7e23209f\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.708144255Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 217.175884ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.709005775Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2caaa990-7bcd-4059-a7c5-3f7de3e3acde\", \"content-length\": \"124\", \"date\": \"Fri, 24 May 2024 15:58:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2caaa990-7bcd-4059-a7c5-3f7de3e3acde\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.927128848Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 313.619089ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:30.927783965Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"386878ed-a65f-4ac4-8d64-2291ba1d3f7e\", \"content-length\": \"124\", \"date\": \"Fri, 24 May 2024 15:58:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"386878ed-a65f-4ac4-8d64-2291ba1d3f7e\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.11281391Z","hostname":"sock","pid":1562,"sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.113429517Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:46335","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196","component":"producer-server","component":"MetricsManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.113461308Z","hostname":"sock","pid":1562,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.113526399Z","hostname":"sock","pid":1562,"file":"oximeter/producer/src/lib.rs:276","component":"producer-server","component":"MetricsManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:276","interval":"30s","address":"[fd00:1122:3344:101::1]:46335","producer_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:31.11962687Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"158.799142ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.158025519Z","hostname":"sock","pid":1562,"component":"VmmReservoirManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.15805608Z","hostname":"sock","pid":1562,"component":"VmmReservoirManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.158133511Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:993","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.158142881Z","hostname":"sock","pid":1562,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.158832759Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.158861299Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:430"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.158887869Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/07053fdc-4cea-499c-875c-c222ca87c041/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.1589017Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/02661193-169e-480c-af92-d631121af19a/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.159164503Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.159358045Z","hostname":"sock","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.159381405Z","hostname":"sock","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.2.100, vlan_id: None }], addresses: [V4(Ipv4Net { addr: 10.151.2.174, width: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.159400686Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.159404606Z","hostname":"sock","pid":1562,"remote_addr":"[fdb0:200::1]:42215","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.347855611Z","hostname":"sock","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:200::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.347907552Z","hostname":"sock","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:200::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.347995613Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/07053fdc-4cea-499c-875c-c222ca87c041/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.348100664Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/02661193-169e-480c-af92-d631121af19a/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.348152055Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.348186365Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.348217475Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.348427908Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:31.349349769Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"661.578954ms"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/f2da503c-9ac6-41a4-a9b4-f11c5d56071c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:31.354087334Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.380101869Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:48924"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.380274451Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.380357201Z","hostname":"sock","pid":1562,"uri":"/inventory","method":"GET","req_id":"82948df3-28e3-48c3-a20b-0e0f8fed6459","remote_addr":"[fd00:1122:3344:101::1]:48924","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":155,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.409010637Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:35774"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.409080218Z","hostname":"sock","pid":1562,"uri":"/sled-role","method":"GET","req_id":"e5431067-7a97-4778-971c-5ee4b96f262a","remote_addr":"[fd00:1122:3344:101::1]:35774","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":35,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.410482884Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/07053fdc-4cea-499c-875c-c222ca87c041/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.410511024Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/02661193-169e-480c-af92-d631121af19a/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.410906519Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.411160792Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:815"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: 54dcf711-c86f-43dc-9f08-95240d2c6844, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, pool_id: 5ed1e6a5-ee1c-433c-a996-15449285d220 (zpool) }, OmicronPhysicalDiskConfig { id: 3466b0bd-eed1-4813-8a6a-b68bf7e16dfa, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, pool_id: f2c30ccb-b1be-4073-8863-4810d7874949 (zpool) }, OmicronPhysicalDiskConfig { id: b83c07ba-9080-4072-9310-4158b6945832, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, pool_id: ef41a08c-320b-4f39-9f50-aa940aa0812c (zpool) }, OmicronPhysicalDiskConfig { id: 7e31e089-5858-449e-9383-23b5f452a013, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, pool_id: 78483afd-3d6d-43ea-a642-d33f5393cede (zpool) }, OmicronPhysicalDiskConfig { id: 1468a190-568a-4f25-bd3f-da7796401754, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, pool_id: d3335b63-f36e-48e0-8ed8-896c0d33a481 (zpool) }, OmicronPhysicalDiskConfig { id: 3b20a792-646a-4572-9905-b014f760fb9a, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, pool_id: 5000f89c-0881-4c9b-a653-9c5490aebd4b (zpool) }, OmicronPhysicalDiskConfig { id: 6cf3c6eb-6cfd-4393-8bde-04a25578536b, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, pool_id: 4c168db8-b979-4ae2-94f5-5b19bad044a5 (zpool) }, OmicronPhysicalDiskConfig { id: 42ca2dfd-26d1-47c3-8629-9058c27c02b7, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, pool_id: b4ac3672-dec0-449f-99d7-e2c61b57bad1 (zpool) }, OmicronPhysicalDiskConfig { id: d8d0bc9f-e0d5-4a8a-bd34-9d09ca172a5d, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, pool_id: 908ee5f1-87cf-4249-8fa1-8af759387a11 (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.44173555Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:347"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.441807411Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:44237"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 54dcf711-c86f-43dc-9f08-95240d2c6844, pool_id: 5ed1e6a5-ee1c-433c-a996-15449285d220 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 3466b0bd-eed1-4813-8a6a-b68bf7e16dfa, pool_id: f2c30ccb-b1be-4073-8863-4810d7874949 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: b83c07ba-9080-4072-9310-4158b6945832, pool_id: ef41a08c-320b-4f39-9f50-aa940aa0812c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 7e31e089-5858-449e-9383-23b5f452a013, pool_id: 78483afd-3d6d-43ea-a642-d33f5393cede (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 1468a190-568a-4f25-bd3f-da7796401754, pool_id: d3335b63-f36e-48e0-8ed8-896c0d33a481 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 3b20a792-646a-4572-9905-b014f760fb9a, pool_id: 5000f89c-0881-4c9b-a653-9c5490aebd4b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 6cf3c6eb-6cfd-4393-8bde-04a25578536b, pool_id: 4c168db8-b979-4ae2-94f5-5b19bad044a5 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 42ca2dfd-26d1-47c3-8629-9058c27c02b7, pool_id: b4ac3672-dec0-449f-99d7-e2c61b57bad1 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: d8d0bc9f-e0d5-4a8a-bd34-9d09ca172a5d, pool_id: 908ee5f1-87cf-4249-8fa1-8af759387a11 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.441988143Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/07053fdc-4cea-499c-875c-c222ca87c041/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.442004413Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/02661193-169e-480c-af92-d631121af19a/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.442011173Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.442016863Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:681"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.442022303Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.442028133Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.442034563Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.442039303Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.548764062Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:31.548794813Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"5ed1e6a5-ee1c-433c-a996-15449285d220 (zpool)"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:32.142170907Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.286557429s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/f2da503c-9ac6-41a4-a9b4-f11c5d56071c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-24T15:58:32.392434296Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:32.592524347Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 5ed1e6a5-ee1c-433c-a996-15449285d220 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:32.777520823Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:32.777649114Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:32.777659304Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:32.777669304Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:32.777677244Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:32.777839526Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:32.777850306Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:33.371001498Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:33.596378095Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:33.789197492Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.546442682s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/f2da503c-9ac6-41a4-a9b4-f11c5d56071c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-24T15:58:33.789266653Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/f2da503c-9ac6-41a4-a9b4-f11c5d56071c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:33.789275293Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Automatically destroying dataset: oxp_5ed1e6a5-ee1c-433c-a996-15449285d220/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:34.022193679Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 60.838554ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:34.207667689Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 117.216025ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:34.747178423Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/f2da503c-9ac6-41a4-a9b4-f11c5d56071c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-24T15:58:35.164046772Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 182.199596ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:35.395066336Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:35.865882036Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.088766721s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 285.81506ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:36.094979697Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/f2da503c-9ac6-41a4-a9b4-f11c5d56071c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:36.096487294Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/f2da503c-9ac6-41a4-a9b4-f11c5d56071c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-24T15:58:36.575755073Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 750.574521ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:36.82244171Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Looking for unencrypted datasets in oxp_5ed1e6a5-ee1c-433c-a996-15449285d220","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:37.073438727Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:37.314207835Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:37.314246386Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:37.314252766Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:37.314258586Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:37.314262636Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:37.5291065Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:37.529141361Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"f2c30ccb-b1be-4073-8863-4810d7874949 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:38.735141155Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: f2c30ccb-b1be-4073-8863-4810d7874949 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:38.991557225Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:38.991606586Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:38.991612746Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:38.991617576Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:38.991644136Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:38.991818098Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:38.991826578Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:39.734434229Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"5.444757927s"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/f2da503c-9ac6-41a4-a9b4-f11c5d56071c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:39.734558321Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/f2da503c-9ac6-41a4-a9b4-f11c5d56071c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-24T15:58:39.734566991Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:40.465752678Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_f2c30ccb-b1be-4073-8863-4810d7874949/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:41.010482353Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/f2da503c-9ac6-41a4-a9b4-f11c5d56071c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-24T15:58:41.257831757Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/f2da503c-9ac6-41a4-a9b4-f11c5d56071c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:41.730905314Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:41.939387224Z","hostname":"sock","pid":1562,"component":"VmmReservoirManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"Looking for unencrypted datasets in oxp_f2c30ccb-b1be-4073-8863-4810d7874949","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.051943641Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.060009435Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.060054706Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.060060876Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.060066316Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.060070566Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.071462739Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.07149219Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"ef41a08c-320b-4f39-9f50-aa940aa0812c (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.169683439Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: ef41a08c-320b-4f39-9f50-aa940aa0812c (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.17827072Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.17830584Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.17831337Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.17835509Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.178362861Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.178521712Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.178529663Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.213761135Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_ef41a08c-320b-4f39-9f50-aa940aa0812c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.222004711Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/f2da503c-9ac6-41a4-a9b4-f11c5d56071c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-24T15:58:42.259906885Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_ef41a08c-320b-4f39-9f50-aa940aa0812c","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.348693214Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.356957351Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.356993291Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.356998711Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.357003341Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.357007351Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.366761025Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.366788926Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"78483afd-3d6d-43ea-a642-d33f5393cede (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.462372494Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 78483afd-3d6d-43ea-a642-d33f5393cede (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.47056038Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.47059465Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.470601871Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.470608271Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.470616911Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.470724932Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.470731402Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.506095906Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_78483afd-3d6d-43ea-a642-d33f5393cede/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.513216869Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_78483afd-3d6d-43ea-a642-d33f5393cede","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.636519783Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.644122451Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.644149632Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.644155702Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.644160272Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.644165352Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.653971417Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.653990427Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"d3335b63-f36e-48e0-8ed8-896c0d33a481 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.749272982Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: d3335b63-f36e-48e0-8ed8-896c0d33a481 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.757082623Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.757116424Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.757121664Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.757152884Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.757161394Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.757293056Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.757300126Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.792149944Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_d3335b63-f36e-48e0-8ed8-896c0d33a481/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.800224608Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_d3335b63-f36e-48e0-8ed8-896c0d33a481","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.922678941Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.930715066Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.930742206Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.930747146Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.930751696Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.930755586Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.940130336Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:42.940149836Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"5000f89c-0881-4c9b-a653-9c5490aebd4b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.032280094Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: 5000f89c-0881-4c9b-a653-9c5490aebd4b (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.040129466Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.040154316Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.040159816Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.040177366Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.040185516Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.040407339Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.040415449Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.061661058Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2303"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.077453223Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_5000f89c-0881-4c9b-a653-9c5490aebd4b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.08581213Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_5000f89c-0881-4c9b-a653-9c5490aebd4b","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.209248395Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.217103627Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.217134787Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.217146447Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.217153287Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.217157267Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.226383895Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.226413796Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"4c168db8-b979-4ae2-94f5-5b19bad044a5 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.321333127Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 4c168db8-b979-4ae2-94f5-5b19bad044a5 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.329236929Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.32929339Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.32930133Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.32931765Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.32932916Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.329465562Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.329479562Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/f2da503c-9ac6-41a4-a9b4-f11c5d56071c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:43.333533349Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/f2da503c-9ac6-41a4-a9b4-f11c5d56071c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-24T15:58:43.33357307Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.366966821Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_4c168db8-b979-4ae2-94f5-5b19bad044a5/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.375122336Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_4c168db8-b979-4ae2-94f5-5b19bad044a5","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.496613148Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.504873805Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.504912395Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.504919665Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.504925875Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.504929635Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.515286657Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.515311627Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"b4ac3672-dec0-449f-99d7-e2c61b57bad1 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.615495819Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: b4ac3672-dec0-449f-99d7-e2c61b57bad1 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.6240552Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.62410268Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.62410962Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.624150011Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.624159771Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.624327733Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.624336643Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.662106585Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_b4ac3672-dec0-449f-99d7-e2c61b57bad1/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.670324811Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_b4ac3672-dec0-449f-99d7-e2c61b57bad1","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.796333866Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.806020059Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.80606474Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.80608928Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.80609646Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.80610292Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.815993996Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.816025606Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"908ee5f1-87cf-4249-8fa1-8af759387a11 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.913641759Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 908ee5f1-87cf-4249-8fa1-8af759387a11 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.922116198Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.922160958Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.922167458Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.922207079Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.922218899Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.922402721Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.922411361Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.95819082Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_908ee5f1-87cf-4249-8fa1-8af759387a11/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:43.966364316Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_908ee5f1-87cf-4249-8fa1-8af759387a11","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.095299014Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.103318098Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.103363009Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/07053fdc-4cea-499c-875c-c222ca87c041/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.103381439Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.103523791Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 07053fdc-4cea-499c-875c-c222ca87c041 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 02661193-169e-480c-af92-d631121af19a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: 5ed1e6a5-ee1c-433c-a996-15449285d220 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: f2c30ccb-b1be-4073-8863-4810d7874949 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: ef41a08c-320b-4f39-9f50-aa940aa0812c (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: 78483afd-3d6d-43ea-a642-d33f5393cede (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: d3335b63-f36e-48e0-8ed8-896c0d33a481 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: 5000f89c-0881-4c9b-a653-9c5490aebd4b (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: 4c168db8-b979-4ae2-94f5-5b19bad044a5 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: b4ac3672-dec0-449f-99d7-e2c61b57bad1 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: 908ee5f1-87cf-4249-8fa1-8af759387a11 (zpool), kind: External } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.103579001Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.103585921Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Writing ledger to /pool/int/02661193-169e-480c-af92-d631121af19a/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.103683222Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.104014996Z","hostname":"sock","pid":1562,"uri":"/omicron-physical-disks","method":"PUT","req_id":"5a97a519-2726-4e9a-9f83-b25442bec25b","remote_addr":"[fd00:1122:3344:101::1]:44237","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":12662170,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 4380d1be-6230-4d40-a925-29519934846c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5ed1e6a5-ee1c-433c-a996-15449285d220 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e8f2ea1-b36e-4377-99fc-17d686c0a2fe, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2c30ccb-b1be-4073-8863-4810d7874949 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1420e4d6-b9c9-4518-a247-528c81f539a8, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef41a08c-320b-4f39-9f50-aa940aa0812c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.13513005Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.135268342Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:37306"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4380d1be-6230-4d40-a925-29519934846c, dataset_name: DatasetName { pool_name: ZpoolName { id: 5ed1e6a5-ee1c-433c-a996-15449285d220 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.135507895Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.135513335Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4380d1be-6230-4d40-a925-29519934846c, dataset_name: DatasetName { pool_name: ZpoolName { id: 5ed1e6a5-ee1c-433c-a996-15449285d220 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e8f2ea1-b36e-4377-99fc-17d686c0a2fe, dataset_name: DatasetName { pool_name: ZpoolName { id: f2c30ccb-b1be-4073-8863-4810d7874949 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.187931728Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.187962949Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e8f2ea1-b36e-4377-99fc-17d686c0a2fe, dataset_name: DatasetName { pool_name: ZpoolName { id: f2c30ccb-b1be-4073-8863-4810d7874949 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1420e4d6-b9c9-4518-a247-528c81f539a8, dataset_name: DatasetName { pool_name: ZpoolName { id: ef41a08c-320b-4f39-9f50-aa940aa0812c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.237715771Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.237758942Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1420e4d6-b9c9-4518-a247-528c81f539a8, dataset_name: DatasetName { pool_name: ZpoolName { id: ef41a08c-320b-4f39-9f50-aa940aa0812c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.287845748Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/07053fdc-4cea-499c-875c-c222ca87c041/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.287880268Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/02661193-169e-480c-af92-d631121af19a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.287888088Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.287925959Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.331572829Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.33161441Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.33162148Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/f2da503c-9ac6-41a4-a9b4-f11c5d56071c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-24T15:58:44.336036662Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_1420e4d6-b9c9-4518-a247-528c81f539a8","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.385126256Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_4380d1be-6230-4d40-a925-29519934846c","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.390236316Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_1e8f2ea1-b36e-4377-99fc-17d686c0a2fe","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.394654928Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_internal_dns_1420e4d6-b9c9-4518-a247-528c81f539a8","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.407743221Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_4380d1be-6230-4d40-a925-29519934846c","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.412393326Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_1e8f2ea1-b36e-4377-99fc-17d686c0a2fe","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:44.416275041Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:45.181173112Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"10.466118875s"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/f2da503c-9ac6-41a4-a9b4-f11c5d56071c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:45.331278569Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/f2da503c-9ac6-41a4-a9b4-f11c5d56071c): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-24T15:58:45.337779375Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Profile for oxz_internal_dns_4380d1be-6230-4d40-a925-29519934846c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:45.948975268Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:45.949089779Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"Profile for oxz_internal_dns_1420e4d6-b9c9-4518-a247-528c81f539a8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:46.428262697Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:46.428368308Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_4380d1be-6230-4d40-a925-29519934846c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:46.913080171Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"Profile for oxz_internal_dns_1e8f2ea1-b36e-4377-99fc-17d686c0a2fe:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:46.913188152Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:46.913226493Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:46.913334944Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1420e4d6-b9c9-4518-a247-528c81f539a8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_4380d1be-6230-4d40-a925-29519934846c\") failed: Property not found. retry in 67.739191ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:46.928931106Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_4380d1be-6230-4d40-a925-29519934846c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:46.928978557Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1e8f2ea1-b36e-4377-99fc-17d686c0a2fe","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_4380d1be-6230-4d40-a925-29519934846c\") failed: Property not found. retry in 88.681456ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:47.006396703Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_4380d1be-6230-4d40-a925-29519934846c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1420e4d6-b9c9-4518-a247-528c81f539a8\") failed: Property not found. retry in 45.268655ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:47.051217287Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1420e4d6-b9c9-4518-a247-528c81f539a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1e8f2ea1-b36e-4377-99fc-17d686c0a2fe\") failed: Property not found. retry in 72.388028ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:47.057655433Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1e8f2ea1-b36e-4377-99fc-17d686c0a2fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4380d1be-6230-4d40-a925-29519934846c\") failed: Property not found. retry in 282.983537ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:47.103309677Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_4380d1be-6230-4d40-a925-29519934846c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1420e4d6-b9c9-4518-a247-528c81f539a8\") failed: Property not found. retry in 112.893302ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:47.109473839Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1420e4d6-b9c9-4518-a247-528c81f539a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1e8f2ea1-b36e-4377-99fc-17d686c0a2fe\") failed: Property not found. retry in 143.587897ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:47.394986851Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1e8f2ea1-b36e-4377-99fc-17d686c0a2fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1420e4d6-b9c9-4518-a247-528c81f539a8\") failed: Property not found. retry in 197.0597ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:47.409065945Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1420e4d6-b9c9-4518-a247-528c81f539a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4380d1be-6230-4d40-a925-29519934846c\") failed: Property not found. retry in 451.829901ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:47.423684196Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_4380d1be-6230-4d40-a925-29519934846c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1e8f2ea1-b36e-4377-99fc-17d686c0a2fe\") failed: Property not found. retry in 112.93903ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:47.557070088Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1e8f2ea1-b36e-4377-99fc-17d686c0a2fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1420e4d6-b9c9-4518-a247-528c81f539a8\") failed: Property not found. retry in 458.588189ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:47.621028396Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1420e4d6-b9c9-4518-a247-528c81f539a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1e8f2ea1-b36e-4377-99fc-17d686c0a2fe\") failed: Property not found. retry in 382.677764ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:47.683496617Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1e8f2ea1-b36e-4377-99fc-17d686c0a2fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4380d1be-6230-4d40-a925-29519934846c\") failed: Property not found. retry in 915.816615ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:47.886822767Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_4380d1be-6230-4d40-a925-29519934846c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1e8f2ea1-b36e-4377-99fc-17d686c0a2fe\") failed: Property not found. retry in 734.766284ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:48.076426675Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1e8f2ea1-b36e-4377-99fc-17d686c0a2fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1420e4d6-b9c9-4518-a247-528c81f539a8\") failed: Property not found. retry in 738.438458ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:48.08877294Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1420e4d6-b9c9-4518-a247-528c81f539a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4380d1be-6230-4d40-a925-29519934846c\") failed: Property not found. retry in 1.171877772s","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:48.811263415Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_4380d1be-6230-4d40-a925-29519934846c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1e8f2ea1-b36e-4377-99fc-17d686c0a2fe\") failed: Property not found. retry in 1.41512917s","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:48.820950919Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1e8f2ea1-b36e-4377-99fc-17d686c0a2fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1420e4d6-b9c9-4518-a247-528c81f539a8\") failed: Property not found. retry in 1.015387878s","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:48.835979284Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1420e4d6-b9c9-4518-a247-528c81f539a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:200::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:49.110816411Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2320"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:49.110846731Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe70:118a","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:49.663688101Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2327"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:49.669368088Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2388"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:49.78834587Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2573"} {"msg":"wait for service Some(\"oxz_internal_dns_1420e4d6-b9c9-4518-a247-528c81f539a8\") failed: Property not found. retry in 1.223498738s","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:49.860578306Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1420e4d6-b9c9-4518-a247-528c81f539a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4380d1be-6230-4d40-a925-29519934846c\") failed: Property not found. retry in 957.192129ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:49.991793621Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_4380d1be-6230-4d40-a925-29519934846c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:50.033203566Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2737"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:50.230494425Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2452"} {"msg":"wait for service Some(\"oxz_internal_dns_1e8f2ea1-b36e-4377-99fc-17d686c0a2fe\") failed: Property not found. retry in 1.031370901s","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:50.245568231Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1e8f2ea1-b36e-4377-99fc-17d686c0a2fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:50.514474288Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2487"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:50.708302376Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2787"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:50.880661184Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2814"} {"msg":"wait for service Some(\"oxz_internal_dns_4380d1be-6230-4d40-a925-29519934846c\") failed: Property not found. retry in 1.403192757s","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:50.957959498Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_4380d1be-6230-4d40-a925-29519934846c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1420e4d6-b9c9-4518-a247-528c81f539a8\") failed: Property not found. retry in 816.588415ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:51.094699598Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1420e4d6-b9c9-4518-a247-528c81f539a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:51.09911095Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2681"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:51.276259273Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2481"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:51.286591574Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3873","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:51.286629344Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3889"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:51.286639595Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"wait for service Some(\"oxz_internal_dns_1e8f2ea1-b36e-4377-99fc-17d686c0a2fe\") failed: Property not found. retry in 952.443757ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:51.28881635Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1e8f2ea1-b36e-4377-99fc-17d686c0a2fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:51.838641245Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3897"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:51.845543265Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3953"} {"msg":"wait for service Some(\"oxz_internal_dns_1420e4d6-b9c9-4518-a247-528c81f539a8\") failed: Property not found. retry in 1.338354536s","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:51.922230473Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_1420e4d6-b9c9-4518-a247-528c81f539a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:51.988103744Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4015"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.095384179Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3996","rack_subnet":"fd00:1122:3344::"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.120012057Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4043"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.240622819Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4070"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.382117595Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:365"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.416613329Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:413"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.448790275Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:651"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.448887176Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:689"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.449296531Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:692","uptime":"3"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.449313281Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:435","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.525669035Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.525757686Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.525772326Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.525780046Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.525787946Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.525795466Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.525809037Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.525816797Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.525824187Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.525831747Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.525844107Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.525851637Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.525859707Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.525882007Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.525888547Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.525895227Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.525901148Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.525907118Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.525912618Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.525918188Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.525922308Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.525926538Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:52.525930748Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.280086484Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_1420e4d6-b9c9-4518-a247-528c81f539a8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.280165295Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_4380d1be-6230-4d40-a925-29519934846c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.280184145Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_1e8f2ea1-b36e-4377-99fc-17d686c0a2fe"} {"msg":"Writing ledger to /pool/int/07053fdc-4cea-499c-875c-c222ca87c041/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.280199925Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/02661193-169e-480c-af92-d631121af19a/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.280535219Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.280940244Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"e5df149f-2c20-42e6-a73c-7102c0441300","remote_addr":"[fd00:1122:3344:101::1]:37306","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":9145600,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.282529393Z","hostname":"sock","pid":1562,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:548"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.330227771Z","hostname":"sock","pid":1562,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:548"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.376923577Z","hostname":"sock","pid":1562,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:548"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.421758432Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:584"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.421770392Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.451807574Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.452579692Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.452599273Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 4380d1be-6230-4d40-a925-29519934846c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5ed1e6a5-ee1c-433c-a996-15449285d220 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1e8f2ea1-b36e-4377-99fc-17d686c0a2fe, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f2c30ccb-b1be-4073-8863-4810d7874949 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 1420e4d6-b9c9-4518-a247-528c81f539a8, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef41a08c-320b-4f39-9f50-aa940aa0812c (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 107379c1-2d45-45dd-a976-3701c49d82f9, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 2ef2416e-6771-48cb-a2d4-ee4b128cb0bb, kind: Service { id: 107379c1-2d45-45dd-a976-3701c49d82f9 }, name: Name(\"ntp-107379c1-2d45-45dd-a976-3701c49d82f9\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 239, 222])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.480949195Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.481073526Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:61047"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4380d1be-6230-4d40-a925-29519934846c, dataset_name: DatasetName { pool_name: ZpoolName { id: 5ed1e6a5-ee1c-433c-a996-15449285d220 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.48139838Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.481414Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4380d1be-6230-4d40-a925-29519934846c, dataset_name: DatasetName { pool_name: ZpoolName { id: 5ed1e6a5-ee1c-433c-a996-15449285d220 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e8f2ea1-b36e-4377-99fc-17d686c0a2fe, dataset_name: DatasetName { pool_name: ZpoolName { id: f2c30ccb-b1be-4073-8863-4810d7874949 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.503146734Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.503196455Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e8f2ea1-b36e-4377-99fc-17d686c0a2fe, dataset_name: DatasetName { pool_name: ZpoolName { id: f2c30ccb-b1be-4073-8863-4810d7874949 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1420e4d6-b9c9-4518-a247-528c81f539a8, dataset_name: DatasetName { pool_name: ZpoolName { id: ef41a08c-320b-4f39-9f50-aa940aa0812c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.523353661Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.523392201Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1420e4d6-b9c9-4518-a247-528c81f539a8, dataset_name: DatasetName { pool_name: ZpoolName { id: ef41a08c-320b-4f39-9f50-aa940aa0812c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.543096682Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/07053fdc-4cea-499c-875c-c222ca87c041/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.543149843Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/02661193-169e-480c-af92-d631121af19a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.543318274Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.543414916Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.554393004Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.586976085Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.587697794Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.587707894Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.630547465Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 239, 222]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.630592236Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"ntp"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.631169603Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.631177443Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.631977252Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.633024794Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.655152143Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:53.672153502Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:54.298993898Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:54.29917474Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9\") failed: Property not found. retry in 26.459455ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:54.434905119Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9\") failed: Property not found. retry in 60.642434ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:54.469085549Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9\") failed: Property not found. retry in 143.706616ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:54.543438119Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9\") failed: Property not found. retry in 413.469188ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:54.699272873Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9\") failed: Property not found. retry in 882.6102ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:55.122517166Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:55.794032705Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:200::2]:64056"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:55.794141116Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"91105511-4c26-4d92-ae8e-3b043aacd8ee","remote_addr":"[fdb0:200::2]:64056","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9\") failed: Property not found. retry in 636.360481ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:56.017639171Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9\") failed: Property not found. retry in 1.298490973s","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:56.663859174Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9\") failed: Property not found. retry in 1.082000416s","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:57.971830771Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9\") failed: Property not found. retry in 722.404442ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:59.062335764Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:59.803917493Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9"} {"msg":"Writing ledger to /pool/int/07053fdc-4cea-499c-875c-c222ca87c041/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:59.803991013Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/02661193-169e-480c-af92-d631121af19a/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:59.804384458Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:59.804695392Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"a629de6c-d8e8-4a6f-9860-bd0adeba8879","remote_addr":"[fd00:1122:3344:101::1]:61047","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":6323554,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:59.80712907Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:625"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:59.837219012Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:59.837354744Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:62056"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:59.84555962Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:58:59.845689831Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"668e2121-4f29-4689-a185-20d77eaf3e9e","remote_addr":"[fd00:1122:3344:101::1]:62056","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8217,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:58:59.847593324Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:00.002591298Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:00.002692919Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:52288"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:04.010900627Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:04.011008948Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"c5433f84-9f29-4610-8d68-5b3b72f2c985","remote_addr":"[fd00:1122:3344:101::1]:52288","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008198,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:04.01464378Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:04.187017278Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:04.187145939Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:53679"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:04.195990233Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:04.196164875Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"d64b8ca2-82e9-4ab4-bb7b-cdbaa18c8f20","remote_addr":"[fd00:1122:3344:101::1]:53679","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8896,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:04.198032106Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:04.351716815Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:04.351813006Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:48180"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_107379c1-2d45-45dd-a976-3701c49d82f9': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:08.359970454Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:08.360117115Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"bbea955f-bc83-46f6-bfd4-8d33cd179cba","remote_addr":"[fd00:1122:3344:101::1]:48180","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008184,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:08.362308121Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:08.536315948Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:08.536453109Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:60169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:08.545761708Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"300682ca-4cb3-459d-8424-56f5b7f2be5d","remote_addr":"[fd00:1122:3344:101::1]:60169","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":9146,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:08.549136278Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:08.549145658Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:08.810974312Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:08.811108584Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:59396"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:08.819184568Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"9e6e63fe-dade-464d-baf1-3128c5fb9b3b","remote_addr":"[fd00:1122:3344:101::1]:59396","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":7923,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:08.82103511Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:08.82104385Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:09.050305743Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:09.050396994Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:45223"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:09.058782302Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"51526768-a713-49a4-8b05-41078147af09","remote_addr":"[fd00:1122:3344:101::1]:45223","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8252,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:09.060706065Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:09.060714395Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:09.305998505Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:09.306118607Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:52544"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:09.314728278Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"00246337-dbc4-4f1f-b80a-b6f30c6c93b7","remote_addr":"[fd00:1122:3344:101::1]:52544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8485,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:09.316549849Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:09.316557459Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:09.60972016Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:09.609818481Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:34365"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:09.618165339Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"4ca2bdcc-b89c-49a5-b38e-c3c9a4f913dc","remote_addr":"[fd00:1122:3344:101::1]:34365","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8261,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:09.61999391Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:09.62001813Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:10.091316346Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:10.091410817Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:33553"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:10.09935708Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"4f1b4032-1a1d-452e-9cab-fa49b33672ba","remote_addr":"[fd00:1122:3344:101::1]:33553","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":7866,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:10.101381704Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:10.101388704Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:10.499875637Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:10.499978898Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:32882"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:10.508725971Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"3843ed18-28ad-4665-8ad4-f3a3ed3e136c","remote_addr":"[fd00:1122:3344:101::1]:32882","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8633,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:10.512291962Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:10.512299363Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:10.685033504Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"350.564434ms"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:11.281796688Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:11.281898229Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:63331"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:11.289768751Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"c97f170b-6109-4baf-9d2f-d9e638e52480","remote_addr":"[fd00:1122:3344:101::1]:63331","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":7758,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:11.291539602Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:11.291547732Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:11.98433673Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:11.984450121Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:39338"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:11.992362514Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"2e318b61-d6bd-4297-a5d7-696e367fa8c8","remote_addr":"[fd00:1122:3344:101::1]:39338","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":7781,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:11.994280456Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:11.994287666Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:12.895083059Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:12.89523561Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:65352"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:12.903384996Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"a723cb4e-1399-4e99-89bc-214ef7732e9b","remote_addr":"[fd00:1122:3344:101::1]:65352","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8116,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:12.905926275Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:12.905933515Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:14.38201301Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:14.382127101Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:41960"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:14.391976816Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"1aa20990-ea43-4f6c-978b-69405f74c228","remote_addr":"[fd00:1122:3344:101::1]:41960","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":9752,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:14.396230336Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:14.396238386Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:15.472259199Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:15.47237569Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:42105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:15.480738068Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"790b006d-2c84-4aef-83c2-36d2b45a3bbc","remote_addr":"[fd00:1122:3344:101::1]:42105","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8226,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:15.483196657Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:15.483214117Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:16.507680346Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:16.507766487Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:60956"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:16.515761401Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"c00371b2-cef5-4788-a917-88e7e59a2ff0","remote_addr":"[fd00:1122:3344:101::1]:60956","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":7880,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:16.517970887Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:16.517977337Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:17.999976021Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:18.000095502Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:39745"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:18.009847446Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"ef501e33-c780-45d3-aa1a-22e4d7d61af6","remote_addr":"[fd00:1122:3344:101::1]:39745","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":9590,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:18.012117463Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:18.012127063Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:21.316212341Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:21.316331072Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:61810"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:21.324984953Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"743a08ef-2c53-4b41-a9af-67cfd7b9f08a","remote_addr":"[fd00:1122:3344:101::1]:61810","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8567,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:21.327361861Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:21.327368591Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:25.131833445Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:25.131955936Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:41574"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:25.141622909Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"85710482-4895-48c4-b001-ce578bcc3888","remote_addr":"[fd00:1122:3344:101::1]:41574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":9521,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:25.145353463Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:25.145361043Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:25.827391615Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:200::2]:46284"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:25.827453866Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"ac20b8d6-7c36-4487-a252-23def7797245","remote_addr":"[fdb0:200::2]:46284","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":33,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:26.068442626Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"291.057457ms"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:28.028193751Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:28.028299312Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:65347"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:28.036880512Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"89f161df-8f8e-46c0-848f-214031935bb3","remote_addr":"[fd00:1122:3344:101::1]:65347","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8408,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:28.039158669Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:28.039166059Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:32.155579804Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:32.155794096Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:51164"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:32.165254727Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"16f60029-fe02-4a69-911b-b63175a77d26","remote_addr":"[fd00:1122:3344:101::1]:51164","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":9395,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:32.167919318Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:32.167926978Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:35.377075175Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:35.377213756Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:45015"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:35.386185512Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"105e238e-6220-46ef-9918-2414fceb088d","remote_addr":"[fd00:1122:3344:101::1]:45015","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8818,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:35.388752862Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:35.388760122Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:38.583133716Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:38.583260147Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:53687"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:38.592393914Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"e4fc2080-7d96-4c95-b486-23218a9d7c5e","remote_addr":"[fd00:1122:3344:101::1]:53687","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8978,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:38.594815532Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:38.594822092Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:41.394005961Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"512.791632ms"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:43.020331354Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:43.020435285Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:50413"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:43.028861924Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"82c2a0b2-3b12-4110-acab-f47c2ca8dd30","remote_addr":"[fd00:1122:3344:101::1]:50413","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8252,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:43.03108381Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:43.03109325Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:52.051913441Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:52.052035832Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:44807"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:52.061770136Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"09b3821a-c6d6-48e4-8554-690609a7122c","remote_addr":"[fd00:1122:3344:101::1]:44807","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":9529,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:52.064039502Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:52.064045963Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:55.858762432Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:200::2]:49593"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:55.85945188Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"467b1fdc-3898-486f-b546-af06f7e6b8ac","remote_addr":"[fdb0:200::2]:49593","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":671,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:56.939406839Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"2.552712174s"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:57.384301845Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:57.384414487Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:59066"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:57.393584394Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"ef4aefec-2574-4c3a-aeda-9e914d2f2e45","remote_addr":"[fd00:1122:3344:101::1]:59066","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":9077,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T15:59:57.396309156Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T15:59:57.396316416Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:00:05.336438089Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:00:05.336551631Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:55994"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:00:05.345825939Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"3e72b2e3-211c-4449-a1c9-748501197d97","remote_addr":"[fd00:1122:3344:101::1]:55994","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":9093,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:00:05.34848119Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T16:00:05.34849048Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T16:00:14.52734693Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"2.651948315s"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:00:20.91193605Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:00:20.912151182Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:44009"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:00:20.921474051Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"c5d38479-fbf8-47bb-bb73-37db40e15fe1","remote_addr":"[fd00:1122:3344:101::1]:44009","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":9256,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:00:20.924066752Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T16:00:20.924075382Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:00:25.89015492Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:200::2]:54065"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:00:25.890237751Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"314369d3-33dc-4b98-ad1c-55176ea55cc8","remote_addr":"[fdb0:200::2]:54065","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":53,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T16:00:32.213800385Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"11.983412224s"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:00:44.382945351Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:00:44.383050782Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:56739"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:00:44.39230215Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"59ff66f0-0052-4071-a661-d30d5b14faac","remote_addr":"[fd00:1122:3344:101::1]:56739","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":9094,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:00:44.394682408Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T16:00:44.395199984Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:00:55.922921643Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:200::2]:47077"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:00:55.922978524Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"ba1c643e-14dd-496c-8713-80d7fa6b296a","remote_addr":"[fdb0:200::2]:47077","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":33,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T16:00:59.232979491Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"19.324857263s"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:00:59.754268752Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:00:59.754383233Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:34902"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:00:59.763174596Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"fd039572-3526-4267-b8b4-f9a5751c4065","remote_addr":"[fd00:1122:3344:101::1]:34902","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8687,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:00:59.765848857Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T16:00:59.765858997Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:01:14.650570973Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:01:14.650687664Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:41407"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:01:14.659687689Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"ca8701a0-e87f-4bd5-86d0-1a4738d0d5d1","remote_addr":"[fd00:1122:3344:101::1]:41407","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8817,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:01:14.662238219Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T16:01:14.662249639Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:01:25.954407901Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:200::2]:61885"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:01:25.954454952Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"d3a7dbd9-e9d8-42c4-83a3-8fed3c89aa3a","remote_addr":"[fdb0:200::2]:61885","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":38,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T16:01:33.591420838Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"40.243338183s"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:01:39.638632768Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:01:39.638740329Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:37941"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:01:39.647830286Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"1a3dce51-ed4c-40eb-bad9-5a5c4eaa0917","remote_addr":"[fd00:1122:3344:101::1]:37941","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":9010,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:01:39.650618128Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T16:01:39.650656649Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:01:55.987575719Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:200::2]:56717"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:01:55.98765572Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"dbbf226b-dcfa-4fb5-8267-6936bb937158","remote_addr":"[fdb0:200::2]:56717","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:02:26.018423081Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:200::2]:57283"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:02:26.018480081Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"60e43e9b-c60f-40b4-983d-4d3552ce6417","remote_addr":"[fdb0:200::2]:57283","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":38,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T16:02:28.868187001Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"44.234977492s"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:02:29.806834837Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:02:29.806948088Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:57954"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:02:29.816214307Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"f6999295-06b2-4904-b535-c4ec988d6a0c","remote_addr":"[fd00:1122:3344:101::1]:57954","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":9083,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:02:29.818879538Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T16:02:29.818887568Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:02:50.799424633Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:02:50.799533214Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:61769"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:02:50.808721891Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"b097613c-caa2-4a4f-aee4-7e6e07f3d819","remote_addr":"[fd00:1122:3344:101::1]:61769","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":9000,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:02:50.811107759Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T16:02:50.811116719Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:02:56.05252211Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:200::2]:43653"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:02:56.053334209Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"1e61c8d9-b5e4-4227-b6a3-60df8a3a41f1","remote_addr":"[fdb0:200::2]:43653","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":794,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:03:26.084772187Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:200::2]:49875"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:03:26.084843328Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"bf6fac6f-24f7-40bb-bea1-7a8b226dc6b5","remote_addr":"[fdb0:200::2]:49875","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T16:03:28.137411519Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"f2da503c-9ac6-41a4-a9b4-f11c5d56071c","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"87.362735975s"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:03:35.061618063Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:03:35.061724624Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:59271"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:03:35.070645049Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"1d2d8c36-bc55-4739-8ee9-8da60b4198c5","remote_addr":"[fd00:1122:3344:101::1]:59271","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8729,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:03:35.07328765Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-24T16:03:35.07329865Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:03:44.104919727Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:03:56.116297286Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:200::2]:33490"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-24T16:03:56.116373387Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"ec95314c-b9a6-4857-9060-84d988c8549c","remote_addr":"[fdb0:200::2]:33490","local_addr":"[fdb0:200::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":51,"response_code":"200"}